v1.4.17rc4

Milestone v2.1: Contract-Driven Architecture & Engineering Hardening

- Stop the Bleeding: version sync checker, PyYAML hardening, install docs unification
- Contract Layer: PFResult/PFError dataclasses, ErrorCode enum, --json wrapping
- Service Extraction: sync.py decomposed into 3 adapters + SyncService
- State Machine: PdfStatus/OcrStatus/Lifecycle enums, transitions, field registry
- Setup Modularization: setup_wizard.py decomposed into 6 classes

173 tests passing, 0 regressions.
This commit is contained in:
Research Assistant 2026-05-09 17:05:07 +08:00
parent 9f511dfd26
commit bbd39a90d3
62 changed files with 4245 additions and 709 deletions

View file

@ -1,24 +1,232 @@
# CI — Plasma Matrix Pipeline (L0-L5 Merge Gate)
#
# Runs on push/PR to main/master. Uses path-filtered triggers for efficient
# resource usage and a re-actors/alls-green aggregator for clean branch protection.
#
# Layer structure:
# L0: Version consistency check (version/__init__.py changes)
# L1: Unit tests — 3 OS x 3 Python plasma matrix
# L2: CLI contract tests — 2 Python x 1 OS
# L3: Plugin tests — Node 20 single config
# L4: E2E + Consistency audit — Python 3.11 single config
# L5: Journey tests — informational, not in merge gate
# Gate: re-actors/alls-green aggregator (excludes L5)
name: CI
on:
"on":
push:
branches: [main, master]
paths-ignore:
- "**.md"
- "docs/**"
pull_request:
branches: [main, master]
paths-ignore:
- "**.md"
- "docs/**"
env:
PYTHONIOENCODING: utf-8
jobs:
plugin-tests:
name: Plugin Tests (L3)
# ---------------------------------------------------------------------------
# Path-filter detection — decides which jobs to run
# ---------------------------------------------------------------------------
changes:
name: Detect Changed Paths
runs-on: ubuntu-latest
outputs:
version: ${{ steps.filter.outputs.version }}
plugin: ${{ steps.filter.outputs.plugin }}
ocr: ${{ steps.filter.outputs.ocr }}
core: ${{ steps.filter.outputs.core }}
any_changed: ${{ steps.filter.outputs.any_changed }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
version:
- 'paperforge/__init__.py'
- 'paperforge/plugin/manifest.json'
- 'paperforge/plugin/versions.json'
- 'CHANGELOG.md'
- 'pyproject.toml'
plugin:
- 'paperforge/plugin/**'
- 'paperforge/plugin/package.json'
ocr:
- 'paperforge/worker/ocr.py'
- 'paperforge/ocr_diagnostics.py'
core:
- 'paperforge/**'
- 'tests/**'
- 'fixtures/**'
- 'pyproject.toml'
- 'scripts/**'
# ---------------------------------------------------------------------------
# L0 — Version consistency check
# ---------------------------------------------------------------------------
version-check:
name: L0 — Version Sync Check
needs: changes
if: ${{ needs.changes.outputs.version == 'true' || needs.changes.outputs.core == 'true' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install package
run: pip install -e .
- name: Check version consistency
run: |
if [ -f scripts/check_version_sync.py ]; then
python scripts/check_version_sync.py
else
python -c "from paperforge import __version__; import json; \
m = json.load(open('paperforge/plugin/manifest.json')); \
assert m['version'] == __version__, \
f'Manifest {m[\"version\"]} != {__version__}'; \
print(f'Version sync: OK ({__version__})')"
fi
# ---------------------------------------------------------------------------
# L1 — Unit tests (plasma matrix: 3 OS x 3 Python)
# ---------------------------------------------------------------------------
unit-tests:
name: L1 — Unit Tests (${{ matrix.os }}, py${{ matrix.python-version }})
needs: changes
if: ${{ needs.changes.outputs.plugin != 'true' || needs.changes.outputs.core == 'true' }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.10", "3.11", "3.12"]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "${{ matrix.python-version }}"
- name: Install package with test deps
run: |
pip install -e ".[test]"
- name: Run unit tests (root tests, not layer-marked)
run: |
python -m pytest tests/ \
--ignore=tests/sandbox \
--ignore=tests/cli \
--ignore=tests/e2e \
--ignore=tests/journey \
--ignore=tests/chaos \
--ignore=tests/audit \
-v --tb=short --timeout=60 \
-x
# ---------------------------------------------------------------------------
# L2 — CLI contract tests (2 Python x 1 OS)
# ---------------------------------------------------------------------------
cli-tests:
name: L2 — CLI Contracts (py${{ matrix.python-version }})
needs: changes
if: ${{ needs.changes.outputs.plugin != 'true' || needs.changes.outputs.core == 'true' }}
strategy:
matrix:
python-version: ["3.10", "3.12"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "${{ matrix.python-version }}"
- name: Install package with test deps
run: pip install -e ".[test]"
- name: Run CLI contract tests
run: python -m pytest tests/cli/ -m cli -v --tb=short --timeout=60 -x
# ---------------------------------------------------------------------------
# L3 — Plugin tests (Node 20, single config)
# ---------------------------------------------------------------------------
plugin-tests:
name: L3 — Plugin Tests
needs: changes
if: ${{ needs.changes.outputs.plugin == 'true' || needs.changes.outputs.core == 'true' }}
runs-on: ubuntu-latest
if: contains(join(github.event.head_commit.modified, ' '), 'paperforge/plugin/') || contains(join(github.event.head_commit.added, ' '), 'paperforge/plugin/') || github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: 'npm'
cache: "npm"
cache-dependency-path: paperforge/plugin/package-lock.json
- run: npm ci
working-directory: paperforge/plugin
- run: npx vitest run --reporter=verbose
working-directory: paperforge/plugin
# ---------------------------------------------------------------------------
# L4 — E2E + consistency audit tests (single config)
# ---------------------------------------------------------------------------
e2e-tests:
name: L4 — E2E + Audit
needs: changes
if: ${{ needs.changes.outputs.plugin != 'true' || needs.changes.outputs.core == 'true' }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.11"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "${{ matrix.python-version }}"
- name: Install package with test deps
run: pip install -e ".[test]"
- name: Run E2E tests
run: python -m pytest tests/e2e/ -m e2e -v --tb=short --timeout=120 -x
- name: Run consistency audit tests
run: python -m pytest tests/audit/ -m audit -v --tb=short --timeout=120 -x
# ---------------------------------------------------------------------------
# L5 — Journey tests (informational, NOT in merge gate)
# ---------------------------------------------------------------------------
journey-tests:
name: L5 — Journey Tests
needs: changes
if: ${{ needs.changes.outputs.any_changed == 'true' }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.11"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "${{ matrix.python-version }}"
- name: Install package with test deps
run: pip install -e ".[test]"
- name: Run journey tests
run: python -m pytest tests/journey/ -m journey -v --tb=short --timeout=120
# ---------------------------------------------------------------------------
# Merge gate — single-status aggregator for branch protection
# ---------------------------------------------------------------------------
alls-green:
name: All Checks Passed
if: always()
needs:
- version-check
- unit-tests
- cli-tests
- plugin-tests
- e2e-tests
runs-on: ubuntu-latest
steps:
- uses: re-actors/alls-green@v1
with:
allowed-skips: version-check, plugin-tests
jobs: ${{ toJSON(needs) }}

61
INSTALLATION.md Normal file
View file

@ -0,0 +1,61 @@
# PaperForge Installation Guide
> Canonical install reference. For other docs, see [README.md](README.md).
---
## Method 1: Obsidian Plugin (Recommended)
1. Download the plugin files from the [latest release](https://github.com/LLLin000/PaperForge/releases/latest).
2. Extract and copy the files into `vault/.obsidian/plugins/paperforge/`.
3. Open Obsidian -> Settings -> Community Plugins -> Enable `PaperForge`.
4. Open PaperForge settings and click `Open Installation Wizard`.
5. Follow the 5-step guided setup: Overview -> Directory Config -> Agent & Keys -> Install -> Done.
> The wizard auto-detects Python, Zotero, and Better BibTeX before installation starts.
---
## Method 2: CLI via pip (Developers)
```bash
pip install git+https://github.com/LLLin000/PaperForge.git
```
Then run the headless setup:
```bash
python -m paperforge setup --headless --agent opencode --paddleocr-key <key>
```
For the latest stable release, replace `master` with a version tag:
```bash
pip install git+https://github.com/LLLin000/PaperForge.git@v1.4.17rc3
```
---
## Method 3: AI Agent Setup
Copy the following to your AI agent for a guided headless install:
- English: [docs/ai-agent-setup-guide.md](docs/ai-agent-setup-guide.md)
- 中文: [docs/ai-agent-setup-guide-zh.md](docs/ai-agent-setup-guide-zh.md)
---
## Prerequisites
| Software | Purpose | Get it |
|----------|---------|--------|
| Python 3.10+ | Run PaperForge CLI and backend tasks | https://python.org |
| Zotero | Literature management | https://zotero.org |
| Better BibTeX | Auto-export metadata as JSON | https://retorque.re/zotero-better-bibtex/ |
| PaddleOCR Key | OCR text and layout extraction | https://aistudio.baidu.com/paddleocr |
---
## Post-Installation
After installing, see [AGENTS.md](AGENTS.md) for first-use workflow and detailed command reference.

View file

@ -41,32 +41,7 @@ PaperForge connects the full path from source literature to structured insight.
## Install
### Obsidian Plugin (Recommended)
1. Download the plugin files from the [latest release](https://github.com/LLLin000/PaperForge/releases/latest).
2. Copy them into `vault/.obsidian/plugins/paperforge/`.
3. Enable `PaperForge` in Obsidian community plugins.
4. Open the settings page and click `打开安装向导`.
5. Follow the 5-step wizard: Overview → Directory Config → Agent & Keys → Install → Done.
> The wizard auto-detects Python, Zotero, and Better BibTeX before installation starts.
### Prerequisites
| Software | Purpose | Get it |
|----------|---------|--------|
| Python 3.10+ | Run PaperForge CLI and backend tasks | https://python.org |
| Zotero | Literature management | https://zotero.org |
| Better BibTeX | Auto-export metadata as JSON | https://retorque.re/zotero-better-bibtex/ |
| PaddleOCR Key | OCR text and layout extraction | https://aistudio.baidu.com/paddleocr |
### CLI (Advanced)
```bash
cd /path/to/your/vault
pip install git+https://github.com/LLLin000/PaperForge.git
python -m paperforge setup --headless --agent opencode --paddleocr-key <key>
```
See [INSTALLATION.md](INSTALLATION.md) for the complete installation guide.
## Usage
@ -131,7 +106,7 @@ python -m paperforge setup --headless --agent opencode --paddleocr-key <key>
| Document | Content |
|----------|---------|
| [Setup Guide](docs/setup-guide.md) | Full setup walkthrough |
| [Quick Install](docs/INSTALLATION.md) | Short install path |
| [Installation Guide](INSTALLATION.md) | Full install reference |
| [Post-Install Guide](AGENTS.md) | First-use workflow guide |
| [Changelog](CHANGELOG.md) | Version history |
| [Contributing](CONTRIBUTING.md) | Dev setup and conventions |

View file

@ -44,32 +44,7 @@ PaperForge 不是单纯的 OCR 工具,也不只是 Zotero 到 Obsidian 的搬
## 安装
### Obsidian 插件(推荐)
1. 从 [最新 Release](https://github.com/LLLin000/PaperForge/releases/latest) 下载插件文件。
2. 复制到 `vault/.obsidian/plugins/paperforge/`
3. 在 Obsidian 中启用 `PaperForge`
4. 打开设置页,点击 `打开安装向导`
5. 跟随 5 步向导完成安装:概览 → 目录配置 → Agent 与密钥 → 安装 → 完成。
> 安装向导会自动检测 Python、Zotero 和 Better BibTeX 是否就绪。
### 前置准备
| 软件 | 用途 | 获取 |
|------|------|------|
| Python 3.10+ | 运行 PaperForge CLI 与后端流程 | https://python.org |
| Zotero | 文献管理 | https://zotero.org |
| Better BibTeX | Zotero JSON 自动导出 | https://retorque.re/zotero-better-bibtex/ |
| PaddleOCR Key | OCR 文本与版面提取 | https://aistudio.baidu.com/paddleocr |
### 命令行安装(开发者)
```bash
cd /path/to/your/vault
pip install git+https://github.com/LLLin000/PaperForge.git
python -m paperforge setup --headless --agent opencode --paddleocr-key <key>
```
完整安装指南请参见 [INSTALLATION.md](INSTALLATION.md)。
## 使用方式

View file

@ -1,6 +1,8 @@
# PaperForge Headless Setup — For LLM Agents (简体中文)
> **IMPORTANT**: 直接读取此文件。不要用 WebFetch 或摘要——会丢失关键的 CLI 参数和决策逻辑。
>
> 完整安装参考请见 [INSTALLATION.md](../INSTALLATION.md)。
---

View file

@ -1,6 +1,8 @@
# PaperForge Headless Setup — For LLM Agents (English)
> **IMPORTANT**: Read this file directly. Do NOT use WebFetch or summarization — you will lose critical CLI flags and decision logic.
>
> For the full installation reference, see [INSTALLATION.md](../INSTALLATION.md).
---

View file

@ -1,7 +1,7 @@
{
"id": "paperforge",
"name": "PaperForge",
"version": "1.4.17rc3",
"version": "1.4.17rc4",
"minAppVersion": "1.9.0",
"description": "PaperForge — Zotero literature pipeline. Sync PDFs, run OCR, and read with AI-assisted deep reading.",
"author": "Lin Zhaoxuan",

View file

@ -1,3 +1,3 @@
"""paperforge — PaperForge package."""
__version__ = "1.4.17rc3"
__version__ = "1.4.17rc4"

View file

216
paperforge/adapters/bbt.py Normal file
View file

@ -0,0 +1,216 @@
from __future__ import annotations
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tuple[str, str, str]:
"""Normalize a BBT attachment path to a consistent storage: format.
Handles three real-world BBT export formats:
1. Absolute Windows paths: D:\\...\\Zotero\\storage\\8CHARKEY\\filename.pdf
-> storage:8CHARKEY/filename.pdf
2. storage: prefix: storage:KEY/filename.pdf -> pass through
3. Bare relative: KEY/filename.pdf -> storage:KEY/filename.pdf
Args:
path: Raw path from BBT JSON attachment.
zotero_dir: Optional absolute path to Zotero data directory for
validating absolute paths.
Returns:
Tuple of (normalized_path, bbt_path_raw, zotero_storage_key).
normalized_path uses forward slashes and storage: prefix for
Zotero storage paths. bbt_path_raw preserves the original input
for debugging. zotero_storage_key is the 8-character Zotero key.
"""
raw = str(path or "").strip()
if not raw:
return ("", "", "")
bbt_path_raw = raw
# Format 2: Already has storage: prefix — pass through with slash normalization
if raw.startswith("storage:"):
storage_rel = raw[len("storage:") :].lstrip("/").lstrip("\\")
storage_rel = storage_rel.replace("\\", "/")
parts = storage_rel.split("/")
zotero_storage_key = parts[0] if parts else ""
return (f"storage:{storage_rel}", bbt_path_raw, zotero_storage_key)
# Format 1: Absolute Windows path pointing to Zotero storage
candidate = Path(raw)
if candidate.is_absolute():
norm_path = raw.replace("\\", "/")
# Detect Zotero storage pattern: .../storage/8CHARKEY/...
if "/storage/" in norm_path:
parts_after_storage = norm_path.split("/storage/", 1)[1]
parts = parts_after_storage.split("/")
if len(parts) >= 2 and len(parts[0]) == 8 and parts[0].isalnum():
zotero_storage_key = parts[0]
filename = "/".join(parts[1:])
return (f"storage:{zotero_storage_key}/{filename}", bbt_path_raw, zotero_storage_key)
# Absolute path but not in Zotero storage — mark as absolute
return (f"absolute:{raw}", bbt_path_raw, "")
# Format 3: Bare relative path — prepend storage: prefix
norm = raw.replace("\\", "/")
parts = norm.split("/")
zotero_storage_key = parts[0] if parts else ""
return (f"storage:{norm}", bbt_path_raw, zotero_storage_key)
def _identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict]]:
"""Identify the main PDF and supplementary materials from attachments.
Uses a hybrid three-priority strategy (Decision D-02):
1. Primary: attachment.title == "PDF" AND contentType == "application/pdf"
2. Fallback heuristic: largest file by size (if available), else shortest title
3. Final fallback: first PDF attachment in the list
Args:
attachments: List of attachment dicts from load_export_rows().
Returns:
Tuple of (main_pdf_attachment, supplementary_attachments).
main_pdf_attachment may be None if no PDFs found.
supplementary_attachments is a list of all other PDF attachments.
"""
pdf_attachments = [a for a in attachments if isinstance(a, dict) and a.get("contentType") == "application/pdf"]
if not pdf_attachments:
return (None, [])
# Priority 1: Title exactly equals "PDF"
for att in pdf_attachments:
if att.get("title") == "PDF":
main = att
supplementary = [a for a in pdf_attachments if a is not main]
return (main, supplementary)
# Priority 2: Largest file by size (if size field is available and differentiated)
sized = [(a, a.get("size", 0) or 0) for a in pdf_attachments]
sized.sort(key=lambda x: x[1], reverse=True)
if sized and sized[0][1] > 0 and (len(sized) == 1 or sized[0][1] > sized[1][1]):
main = sized[0][0]
supplementary = [a for a in pdf_attachments if a is not main]
return (main, supplementary)
# Priority 2b (sizes equal or unavailable): shortest title
titled = [(a, len(str(a.get("title", "")))) for a in pdf_attachments]
titled.sort(key=lambda x: x[1])
main = titled[0][0]
supplementary = [a for a in pdf_attachments if a is not main]
return (main, supplementary)
def extract_authors(item: dict) -> list[str]:
authors = []
for creator in item.get("creators", []):
if creator.get("creatorType") != "author":
continue
full_name = " ".join(
part for part in [creator.get("firstName", ""), creator.get("lastName", "")] if part
).strip()
if full_name:
authors.append(full_name)
elif creator.get("name"):
authors.append(creator["name"])
return authors
def collection_fields(collection_paths: list[str]) -> dict[str, str | list[str]]:
paths = [path for path in collection_paths if path]
primary = paths[0] if paths else ""
if paths:
primary = sorted(paths, key=lambda value: (value.count("/"), len(value), value), reverse=True)[0]
tags = []
seen = set()
for path in paths:
for part in [segment.strip() for segment in path.split("/") if segment.strip()]:
if part not in seen:
seen.add(part)
tags.append(part)
group = primary
return {"collections": paths, "collection_tags": tags, "collection_group": [group] if group else []}
def resolve_item_collection_paths(item: dict, collection_lookup: dict) -> list[str]:
paths = []
collection_keys = item.get("collections") or []
if collection_keys:
for key in collection_keys:
paths.append(collection_lookup.get("path_by_key", {}).get(key, key))
item_id = item.get("itemID")
if item_id is not None:
paths.extend(collection_lookup.get("paths_by_item_id", {}).get(item_id, []))
return sorted({path for path in paths if path}, key=lambda value: (-value.count("/"), value))
def load_export_rows(path: Path) -> list[dict]:
from paperforge.worker._domain import build_collection_lookup
from paperforge.worker._utils import _extract_year, read_json
data = read_json(path)
if isinstance(data, list):
return data
if isinstance(data, dict) and isinstance(data.get("items"), list):
collection_lookup = build_collection_lookup(data.get("collections", {}))
rows = []
for item in data["items"]:
if item.get("itemType") in {"attachment", "note", "annotation"}:
continue
attachments = []
for attachment in item.get("attachments", []):
if not isinstance(attachment, dict):
continue
raw_path = attachment.get("path", "")
normalized_path, bbt_path_raw, zotero_storage_key = _normalize_attachment_path(raw_path)
# Preserve contentType from BBT if present; fallback to file extension
content_type = attachment.get("contentType", "")
if not content_type and str(normalized_path).lower().endswith(".pdf"):
content_type = "application/pdf"
attachments.append(
{
"path": normalized_path,
"contentType": content_type,
"title": attachment.get("title", ""),
"bbt_path_raw": bbt_path_raw,
"zotero_storage_key": zotero_storage_key,
"size": attachment.get("size", 0) or 0,
}
)
main_pdf, supplementary_pdfs = _identify_main_pdf(attachments)
pdf_path = main_pdf["path"] if main_pdf else ""
bbt_path_raw = main_pdf["bbt_path_raw"] if main_pdf else ""
zotero_storage_key = main_pdf["zotero_storage_key"] if main_pdf else ""
path_error = "not_found" if not main_pdf else ""
supplementary = [a["path"] for a in supplementary_pdfs] if supplementary_pdfs else []
attachment_count = len(attachments)
rows.append(
{
"key": item.get("key") or item.get("itemKey", ""),
"title": item.get("title", ""),
"authors": extract_authors(item),
"creators": item.get("creators", []),
"abstract": item.get("abstractNote", ""),
"journal": item.get("publicationTitle", ""),
"extra": item.get("extra", ""),
"year": _extract_year(item.get("date", "")),
"date": item.get("date", ""),
"doi": item.get("DOI", ""),
"pmid": item.get("PMID", ""),
"collections": resolve_item_collection_paths(item, collection_lookup),
"attachments": attachments,
"pdf_path": pdf_path,
"supplementary": supplementary,
"attachment_count": attachment_count,
"bbt_path_raw": bbt_path_raw,
"zotero_storage_key": zotero_storage_key,
"path_error": path_error,
}
)
return rows
raise ValueError(f"Unsupported export format: {path}")

View file

@ -0,0 +1,318 @@
from __future__ import annotations
import re
from pathlib import Path
from typing import Optional
def _yaml_quote(value: str) -> str:
if isinstance(value, bool):
return "true" if value else "false"
return '"' + str(value or "").replace("\\", "\\\\").replace('"', '\\"') + '"'
def _yaml_block(value: str) -> list[str]:
value = (value or "").strip()
if not value:
return ["abstract: |-", " "]
lines = ["abstract: |-"]
for line in value.splitlines():
lines.append(f" {line}")
return lines
def compute_final_collection(row: dict) -> str:
user_raw = str(row.get("user_collection", "") or "").strip()
user_resolved = str(row.get("user_collection_resolved", "") or "").strip()
recommended = str(row.get("recommended_collection", "") or "").strip()
if user_raw:
return user_resolved
return recommended
DEEP_READING_HEADER = "## 🔍 精读"
def _read_frontmatter_bool_from_text(text: str, key: str, default: bool = False) -> bool:
match = re.search(rf"^{re.escape(key)}:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE)
if not match:
return default
return match.group(1).lower() == "true"
def _read_frontmatter_optional_bool_from_text(text: str, key: str) -> Optional[bool]:
match = re.search(rf"^{re.escape(key)}:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE)
if not match:
return None
return match.group(1).lower() == "true"
def _legacy_control_flags(paths: dict[str, Path], zotero_key: str) -> dict[str, Optional[bool]]:
records_root = paths.get("library_records")
if not records_root or not records_root.exists():
return {"do_ocr": None, "analyze": None}
for record_path in records_root.rglob(f"{zotero_key}.md"):
try:
text = record_path.read_text(encoding="utf-8")
except Exception:
continue
return {
"do_ocr": _read_frontmatter_optional_bool_from_text(text, "do_ocr"),
"analyze": _read_frontmatter_optional_bool_from_text(text, "analyze"),
}
return {"do_ocr": None, "analyze": None}
def canonicalize_decision(value: str) -> str:
text = str(value or "").strip()
if text in {"", "待查"}:
return "待定"
if text in {"排除", "不纳入"}:
return "不纳入"
if text == "纳入":
return "纳入"
return "待定"
def candidate_markdown(row: dict) -> str:
row = dict(row)
row["final_collection"] = compute_final_collection(row)
row["decision"] = canonicalize_decision(row.get("decision", ""))
lines = ["---"]
ordered_keys = [
"candidate_id",
"domain",
"title",
"authors",
"year",
"journal",
"doi",
"pmid",
"source",
"requester_skill",
"request_context",
"abstract_short",
"decision",
"recommended_collection",
"recommend_confidence",
"recommend_reason",
"user_collection",
"user_collection_resolved",
"final_collection",
"collection_resolution",
"duplicate_hint",
"existing_zotero_key",
"existing_collections",
"import_status",
"note",
"candidate_source_type",
"source_zotero_key",
"cited_ref_number",
"trigger_sentence",
"source_context",
"task_relevance_reason",
"harvest_priority",
"raw_reference",
"status",
]
row.setdefault("status", "candidate")
for key in ordered_keys:
value = row.get(key, "")
if isinstance(value, list):
lines.append(f"{key}:")
for item in value:
lines.append(f" - {_yaml_quote(item)}")
elif value == "":
lines.append(f"{key}:")
elif "\n" in str(value):
lines.extend(
_yaml_block(str(value)).copy()
if key == "abstract"
else [f"{key}: |-"] + [f" {line}" for line in str(value).splitlines()]
)
else:
lines.append(f"{key}: {_yaml_quote(value)}")
lines.extend(
[
"---",
"",
f"# {row['candidate_id']}",
"",
"候选文献轻量记录,仅用于 Base 决策和 write-back 触发,不是正式文献卡片。",
"",
]
)
return "\n".join(lines)
def generate_review(candidates: list[dict]) -> str:
normalized = []
for row in candidates:
copy = dict(row)
copy["decision"] = canonicalize_decision(copy.get("decision", ""))
normalized.append(copy)
include = [c for c in normalized if c.get("decision") == "纳入"]
exclude = [c for c in normalized if c.get("decision") == "不纳入"]
lines = [
"# 本轮候选总览",
"",
"## 检索背景",
"",
f"- 候选数量:{len(normalized)}",
f"- 建议纳入:{len(include)}",
f"- 不纳入:{len(exclude)}",
"",
"## 总体判断",
"",
"- 当前候选池已经按决策状态分层,可直接进入 Base 处理。",
"",
"## 推荐优先纳入",
"",
]
if include:
for row in include:
lines.extend(
[
f"### {row['candidate_id']}",
"",
f"- 标题:{row['title']}",
f"- 推荐分类:`{compute_final_collection(row)}`",
f"- 理由:{row.get('recommend_reason', '')}",
"",
]
)
else:
lines.extend(["- 暂无", ""])
lines.extend(["## 不纳入", ""])
if exclude:
for row in exclude:
lines.extend(
[
f"### {row['candidate_id']}",
"",
f"- 标题:{row['title']}",
f"- 理由:{row.get('recommend_reason', '')}",
"",
]
)
else:
lines.extend(["- 暂无", ""])
lines.extend(["## 下一步", "", "1. 在 Base 中确认决策。", "2. 对纳入项执行 write-back。", "3. 刷新正式索引。", ""])
return "\n".join(lines)
def extract_preserved_deep_reading(text: str) -> str:
if not text:
return ""
match = re.search("^## 🔍 精读\\s*$", text, re.MULTILINE)
if not match:
return ""
start = match.start()
preserved = text[start:].strip()
return preserved
def has_deep_reading_content(text: str) -> bool:
preserved = extract_preserved_deep_reading(text)
if not preserved:
return False
body = preserved.replace(DEEP_READING_HEADER, "").strip()
if not body:
return False
clarity_ok = bool(re.search(
r'-\s*\*\*Clarity\*\*(清晰度):(.+)', body
)) and not re.search(
r'-\s*\*\*Clarity\*\*(清晰度):\s*$', body, re.MULTILINE
)
figure_sec = _extract_section(body, r'\*\*Figure 导读\*\*')
figure_ok = False
if figure_sec:
for line in figure_sec.splitlines():
s = line.strip()
if s.startswith('- ') and '' in s:
_, after = s.split('', 1)
if after.strip() and after.strip() != '(待补充)':
figure_ok = True
break
issue_sec = _extract_section(body, r'\*\*遗留问题\*\*')
if not issue_sec:
issue_sec = _extract_section(body, r'####\s*遗留问题')
issue_ok = False
if issue_sec:
dirty = [l.strip() for l in issue_sec.splitlines() if l.strip()]
substantive = [l for l in dirty if l not in ('-', '(待补充)', '', '**遗留问题**')]
issue_ok = bool(substantive)
return clarity_ok and figure_ok and issue_ok
def _extract_section(body: str, section_header: str) -> str | None:
m = re.search(
section_header + r'\n*(.*?)(?=\n(?:#{1,3})\s|\Z)',
body, re.DOTALL,
)
if m:
return m.group(1).strip()
return None
def _add_missing_frontmatter_fields(existing_content: str, new_fields: dict[str, str]) -> str:
if not existing_content.startswith("---"):
return existing_content
parts = existing_content.split("---", 2)
if len(parts) < 3:
return existing_content
frontmatter = parts[1]
body = parts[2]
lines_to_add = []
for key, value in new_fields.items():
pattern = "^" + re.escape(key) + "\\s*:"
if not re.search(pattern, frontmatter, re.MULTILINE):
lines_to_add.append(f"{key}: {_yaml_quote(value)}")
if not lines_to_add:
return existing_content
new_frontmatter = frontmatter.rstrip("\n") + "\n" + "\n".join(lines_to_add) + "\n"
return f"---{new_frontmatter}---{body}"
def update_frontmatter_field(content: str, key: str, value: str) -> str:
if not content.startswith("---"):
return content
pattern = "^" + re.escape(key) + "\\s*:.*$"
replacement = f"{key}: {_yaml_quote(value)}"
new_content, count = re.subn(pattern, replacement, content, flags=re.MULTILINE, count=1)
if count == 0:
new_content = _add_missing_frontmatter_fields(content, {key: value})
return new_content
def read_frontmatter_dict(text: str) -> dict:
"""Parse YAML frontmatter from markdown text using yaml.safe_load.
Falls back to regex parsing if YAML parser fails (malformed frontmatter).
Returns empty dict if no frontmatter.
"""
import re
import yaml
fm_match = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
if not fm_match:
return {}
try:
data = yaml.safe_load(fm_match.group(1))
if isinstance(data, dict):
return data
except Exception:
pass
result = {}
for line in fm_match.group(1).splitlines():
line = line.strip()
if ':' in line:
key, _, val = line.partition(':')
result[key.strip()] = val.strip()
return result

View file

@ -0,0 +1,62 @@
from __future__ import annotations
import os
from pathlib import Path
def obsidian_wikilink_for_pdf(pdf_path: str, vault_dir: Path, zotero_dir: Path | None = None) -> str:
text = str(pdf_path or "").strip()
if not text:
return ""
# Handle storage: prefix paths by resolving through zotero_dir
if text.startswith("storage:") and zotero_dir is not None:
storage_rel = text[len("storage:") :].lstrip("/").lstrip("\\")
absolute_pdf_path = zotero_dir / "storage" / storage_rel.replace("/", os.sep)
absolute_str = str(absolute_pdf_path)
else:
absolute_str = absolutize_vault_path(vault_dir, text, resolve_junction=True)
if not absolute_str:
return ""
absolute_path = Path(absolute_str)
try:
relative = absolute_path.relative_to(vault_dir)
except ValueError:
# Path outside vault — try to route through Zotero junction inside vault
if zotero_dir is not None and zotero_dir.exists():
try:
from paperforge.pdf_resolver import resolve_junction
real_zotero = resolve_junction(zotero_dir)
if real_zotero != zotero_dir:
rel_to_zotero = absolute_path.relative_to(real_zotero)
via_junction = zotero_dir / rel_to_zotero
relative = via_junction.relative_to(vault_dir)
return f"[[{relative.as_posix()}]]"
except (ValueError, OSError):
pass
return f"[[{absolute_path.as_posix()}]]"
return f"[[{relative.as_posix()}]]"
def absolutize_vault_path(vault: Path, path: str, resolve_junction: bool = False) -> str:
text = str(path or "").strip()
if not text:
return ""
candidate = Path(text)
result = str(candidate) if candidate.is_absolute() else str((vault / text.replace("/", os.sep)).resolve())
if resolve_junction:
from paperforge.pdf_resolver import resolve_junction
result = str(resolve_junction(Path(result)))
return result
def obsidian_wikilink_for_path(vault: Path, path: str) -> str:
absolute = absolutize_vault_path(vault, path, resolve_junction=True)
if not absolute:
return ""
absolute_path = Path(absolute)
try:
relative = absolute_path.relative_to(vault)
except ValueError:
return f"[[{absolute_path.as_posix()}]]"
return f"[[{relative.as_posix()}]]"

View file

@ -181,6 +181,11 @@ def build_parser() -> argparse.ArgumentParser:
action="store_true",
help="Force full rebuild of the canonical asset index",
)
p_sync.add_argument(
"--json",
action="store_true",
help="Output result as JSON (PFResult envelope)",
)
# selection-sync (backward compat)
sub.add_parser("selection-sync", help="Sync Zotero selection to library records")
@ -203,6 +208,11 @@ def build_parser() -> argparse.ArgumentParser:
action="store_true",
help="Diagnose OCR configuration without running",
)
p_ocr.add_argument(
"--json",
action="store_true",
help="Output result as JSON (PFResult envelope)",
)
p_ocr.add_argument(
"--key",
metavar="KEY",
@ -237,6 +247,10 @@ def build_parser() -> argparse.ArgumentParser:
help="Output all entries in the canonical index (JSON array)",
)
# dashboard
p_dash = sub.add_parser("dashboard", help="Aggregated stats and permissions for the plugin dashboard")
p_dash.add_argument("--json", action="store_true", help="Output as PFResult JSON")
# base-refresh
p_base = sub.add_parser("base-refresh", help="Refresh Obsidian Base view files")
p_base.add_argument(
@ -247,7 +261,8 @@ def build_parser() -> argparse.ArgumentParser:
)
# doctor
sub.add_parser("doctor", help="Validate PaperForge setup and configuration")
doctor_p = sub.add_parser("doctor", help="Validate PaperForge setup and configuration")
doctor_p.add_argument("--json", action="store_true", dest="json_output", help="Output JSON")
# update
sub.add_parser("update", help="Update PaperForge to the latest version")
@ -312,6 +327,11 @@ def build_parser() -> argparse.ArgumentParser:
action="store_true",
help="Skip environment checks (for testing/CI)",
)
p_setup.add_argument(
"--modular",
action="store_true",
help="Use modular setup components (v2.1+)",
)
return parser
@ -438,6 +458,11 @@ def main(argv: list[str] | None = None) -> int:
return repair.run(args)
if args.command == "dashboard":
from paperforge.commands import dashboard
return dashboard.run(args)
if args.command == "base-refresh":
force = getattr(args, "force", False)
paths = args.paths
@ -450,7 +475,12 @@ def main(argv: list[str] | None = None) -> int:
if args.command == "doctor":
from paperforge.worker.status import run_doctor
return run_doctor(vault)
kw = {}
if getattr(args, "verbose", False):
kw["verbose"] = True
if getattr(args, "json_output", False):
kw["json_output"] = True
return run_doctor(vault, **kw)
if args.command == "update":
from paperforge.worker.update import run_update
@ -458,7 +488,23 @@ def main(argv: list[str] | None = None) -> int:
return run_update(vault)
if args.command == "setup":
if getattr(args, "headless", False):
if getattr(args, "modular", False):
from paperforge.setup.plan import SetupPlan
config = {
"system_dir": getattr(args, "system_dir", None) or "System",
"resources_dir": getattr(args, "resources_dir", None) or "Resources",
"literature_dir": getattr(args, "literature_dir", None) or "Literature",
"control_dir": getattr(args, "control_dir", None) or "LiteratureControl",
"base_dir": getattr(args, "base_dir", None) or "Bases",
}
plan = SetupPlan(
vault=vault,
config=config,
agent_type=getattr(args, "agent", "opencode"),
)
return plan.execute(json_output=getattr(args, "json_output", False))
elif getattr(args, "headless", False):
from paperforge.setup_wizard import headless_setup
return headless_setup(

View file

@ -8,6 +8,7 @@ _COMMAND_REGISTRY: dict[str, str] = {
"repair": "paperforge.commands.repair",
"status": "paperforge.commands.status",
"context": "paperforge.commands.context",
"dashboard": "paperforge.commands.dashboard",
}

View file

@ -0,0 +1,167 @@
"""Dashboard command.
Returns a PFResult with aggregated stats and permissions for the plugin dashboard.
"""
from __future__ import annotations
import logging
import os
import re
from pathlib import Path
from paperforge import __version__
from paperforge.config import load_vault_config, paperforge_paths, resolve_vault
from paperforge.core.errors import ErrorCode
from paperforge.core.result import PFError, PFResult
logger = logging.getLogger(__name__)
def run(args) -> int:
"""Run dashboard command and print PFResult JSON to stdout."""
vault = getattr(args, "vault_path", None)
if vault is None:
try:
vault = resolve_vault(cli_vault=getattr(args, "vault", None))
except FileNotFoundError as exc:
result = PFResult(
ok=False,
command="dashboard",
version=__version__,
error=PFError(code=ErrorCode.INTERNAL_ERROR, message=str(exc)),
)
print(result.to_json())
return 1
try:
data = _gather_dashboard_data(vault)
result = PFResult(ok=True, command="dashboard", version=__version__, data=data)
print(result.to_json())
return 0
except Exception as exc:
logger.exception("Dashboard gathering failed")
result = PFResult(
ok=False,
command="dashboard",
version=__version__,
error=PFError(code=ErrorCode.INTERNAL_ERROR, message=str(exc)),
)
print(result.to_json())
return 1
def _gather_dashboard_data(vault: Path) -> dict:
"""Gather stats and permissions for dashboard display."""
cfg = load_vault_config(vault)
paths = paperforge_paths(vault, cfg)
# ── Papers / formal note count ──
_skip_names = {"fulltext.md", "deep-reading.md", "discussion.md"}
record_count = 0
if paths["literature"].exists():
for p in paths["literature"].rglob("*.md"):
if p.name not in _skip_names:
record_count += 1
# ── Domain counts (first-level subdirs under literature) ──
domain_counts: dict[str, int] = {}
if paths["literature"].exists():
for domain_dir in sorted(paths["literature"].iterdir()):
if domain_dir.is_dir():
count = sum(
1 for p in domain_dir.rglob("*.md") if p.name not in _skip_names
)
if count > 0:
domain_counts[domain_dir.name] = count
# ── PDF health & OCR health from frontmatter ──
pdf_healthy = 0
pdf_broken = 0
pdf_missing = 0
ocr_pending = 0
ocr_done = 0
ocr_failed = 0
_path_error_pat = re.compile(r'^path_error:\s*"(.+?)"\s*$', re.MULTILINE)
_pdf_path_pat = re.compile(r'^pdf_path:\s*".*?"\s*$', re.MULTILINE)
_ocr_status_pat = re.compile(r'^ocr_status:\s*(\S+)', re.MULTILINE)
_do_ocr_pat = re.compile(r'^do_ocr:\s*true\s*$', re.MULTILINE)
if paths["literature"].exists():
for note_path in paths["literature"].rglob("*.md"):
if note_path.name in _skip_names:
continue
try:
text = note_path.read_text(encoding="utf-8")
except Exception:
continue
# PDF health
path_error_m = _path_error_pat.search(text)
if path_error_m:
error_type = path_error_m.group(1)
if error_type == "not_found":
pdf_missing += 1
else:
pdf_broken += 1
elif _pdf_path_pat.search(text):
pdf_healthy += 1
# OCR health
ocr_status_m = _ocr_status_pat.search(text)
if ocr_status_m:
status = ocr_status_m.group(1).strip().lower().strip('"')
if status == "done":
ocr_done += 1
elif status in ("pending", "queued", "processing", "running", ""):
ocr_pending += 1
else:
ocr_failed += 1
elif _do_ocr_pat.search(text):
ocr_pending += 1
# ── Permissions ──
export_files = sorted(paths["exports"].glob("*.json")) if paths["exports"].exists() else []
can_sync = len(export_files) > 0
paddle_token = (
os.environ.get("PADDLEOCR_API_TOKEN")
or os.environ.get("PADDLEOCR_API_KEY")
or os.environ.get("OCR_TOKEN")
)
can_ocr = bool(paddle_token)
can_copy_context = False
pf_dir = paths.get("paperforge", vault / cfg["system_dir"] / "PaperForge")
if pf_dir.exists():
try:
pf_dir.parent.mkdir(parents=True, exist_ok=True)
test_file = pf_dir / ".write_test"
test_file.touch()
test_file.unlink()
can_copy_context = True
except (OSError, PermissionError):
pass
return {
"stats": {
"papers": record_count,
"pdf_health": {
"healthy": pdf_healthy,
"broken": pdf_broken,
"missing": pdf_missing,
},
"ocr_health": {
"pending": ocr_pending,
"done": ocr_done,
"failed": ocr_failed,
},
"domain_counts": domain_counts,
},
"permissions": {
"can_sync": can_sync,
"can_ocr": can_ocr,
"can_copy_context": can_copy_context,
},
}

View file

@ -4,10 +4,54 @@ import argparse
import logging
from pathlib import Path
from paperforge import __version__
from paperforge.core.errors import ErrorCode
from paperforge.core.result import PFError, PFResult
logger = logging.getLogger(__name__)
def _diagnose(vault: Path, live: bool = False) -> int:
def _collect_ocr_queue_data(vault: Path) -> dict:
"""Scan OCR meta files and build queue status data dict.
Returns dict shaped as:
{queue: {pending: [...], processing: [...], done: [...], failed: [...]},
total: N, done: N, failed: N, pending: N, processing: N}
"""
from paperforge.worker._utils import pipeline_paths, read_json
paths = pipeline_paths(vault)
queue_lists = {"pending": [], "processing": [], "done": [], "failed": []}
ocr_root = paths.get("ocr")
if ocr_root and ocr_root.exists():
for meta_path in sorted(ocr_root.glob("*/meta.json")):
try:
meta = read_json(meta_path)
except Exception:
continue
key = str(meta.get("zotero_key", "") or "").strip()
status = str(meta.get("ocr_status", "") or "").strip().lower()
if not key:
continue
if status == "done":
queue_lists["done"].append(key)
elif status in ("queued", "running", "processing"):
queue_lists["processing"].append(key)
elif status == "pending":
queue_lists["pending"].append(key)
else:
queue_lists["failed"].append(key)
return {
"queue": queue_lists,
"total": sum(len(v) for v in queue_lists.values()),
"done": len(queue_lists["done"]),
"failed": len(queue_lists["failed"]),
"pending": len(queue_lists["pending"]),
"processing": len(queue_lists["processing"]),
}
def _diagnose(vault: Path, live: bool = False, json_output: bool = False) -> int:
"""Run OCR diagnostics and print results."""
from paperforge.ocr_diagnostics import ocr_doctor
@ -15,6 +59,32 @@ def _diagnose(vault: Path, live: bool = False) -> int:
level = result.get("level", 0)
passed = result.get("passed", False)
if json_output:
queue_data = _collect_ocr_queue_data(vault)
pf_error = None
if not passed:
pf_error = PFError(
code=ErrorCode.OCR_TOKEN_MISSING if level == 1 else ErrorCode.INTERNAL_ERROR,
message=result.get("error", "OCR diagnosis failed"),
details={"level": level, "fix": result.get("fix", "")},
)
pf_result = PFResult(
ok=passed,
command="ocr-diagnose",
version=__version__,
data={
"diagnosis": {
"level": level,
"passed": passed,
"message": result.get("message", result.get("error", "")),
},
**queue_data,
},
error=pf_error,
)
print(pf_result.to_json())
return 0 if passed else 1
print(f"OCR Doctor — Level {level} diagnostic")
print("-" * 40)
if passed:
@ -64,11 +134,12 @@ def run(args: argparse.Namespace) -> int:
diagnose_only = getattr(args, "diagnose", False)
key = getattr(args, "key", None)
live = getattr(args, "live", False)
json_output = getattr(args, "json", False)
# Backward compat: if subcommand was "doctor", diagnose
ocr_action = getattr(args, "ocr_action", None)
if ocr_action == "doctor" or diagnose_only:
return _diagnose(vault, live=live)
return _diagnose(vault, live=live, json_output=json_output)
if key:
logger.info("Processing specific key: %s", key)

View file

@ -1,10 +1,14 @@
"""Sync command — unifies selection-sync and index-refresh."""
import argparse
import json as _json
import logging
from pathlib import Path
from paperforge import __version__
from paperforge.config import migrate_paperforge_json
from paperforge.core.errors import ErrorCode
from paperforge.core.result import PFError, PFResult
logger = logging.getLogger(__name__)
@ -74,8 +78,18 @@ def run(args: argparse.Namespace) -> int:
index_only = getattr(args, "index", False)
rebuild_index = getattr(args, "rebuild_index", False)
domain = getattr(args, "domain", None)
json_output = getattr(args, "json", False)
if dry_run:
if json_output:
result = PFResult(
ok=True,
command="sync",
version=__version__,
data={"dry_run": True, "selection": not index_only, "index": not selection_only},
)
print(result.to_json())
return 0
print("[DRY-RUN] Would run sync operations")
if not selection_only and not index_only:
print(" - selection-sync")
@ -90,17 +104,39 @@ def run(args: argparse.Namespace) -> int:
return 0
exit_code = 0
sync_counts = {"new": 0, "updated": 0, "skipped": 0, "failed": 0, "errors": []}
if not index_only:
run_selection_sync = _get_run_selection_sync()
code = run_selection_sync(vault, verbose=getattr(args, "verbose", False))
if code != 0:
exit_code = code
result = run_selection_sync(vault, verbose=getattr(args, "verbose", False), json_output=json_output)
if json_output:
sync_counts = result
elif result != 0:
exit_code = result
if not selection_only:
run_index_refresh = _get_run_index_refresh()
code = run_index_refresh(vault, verbose=getattr(args, "verbose", False), rebuild_index=rebuild_index)
code = run_index_refresh(vault, verbose=getattr(args, "verbose", False), rebuild_index=rebuild_index, json_output=json_output)
if code != 0 and exit_code == 0:
exit_code = code
if json_output:
errors = sync_counts.get("errors", [])
pf_error = None
if errors or exit_code != 0:
pf_error = PFError(
code=ErrorCode.SYNC_FAILED,
message=f"Sync completed with {len(errors)} error(s)",
details={"errors": errors, "exit_code": exit_code},
)
result = PFResult(
ok=exit_code == 0 and not errors,
command="sync",
version=__version__,
data=sync_counts,
error=pf_error,
)
print(result.to_json())
return 0
return exit_code

View file

19
paperforge/core/errors.py Normal file
View file

@ -0,0 +1,19 @@
from __future__ import annotations
from enum import Enum
class ErrorCode(str, Enum):
"""Centralized error codes for PFResult contracts."""
PYTHON_NOT_FOUND = "PYTHON_NOT_FOUND"
VERSION_MISMATCH = "VERSION_MISMATCH"
BBT_EXPORT_NOT_FOUND = "BBT_EXPORT_NOT_FOUND"
OCR_TOKEN_MISSING = "OCR_TOKEN_MISSING"
SYNC_FAILED = "SYNC_FAILED"
VALIDATION_ERROR = "VALIDATION_ERROR"
INTERNAL_ERROR = "INTERNAL_ERROR"
UNKNOWN = "UNKNOWN"
def __str__(self) -> str:
return self.value

67
paperforge/core/result.py Normal file
View file

@ -0,0 +1,67 @@
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import Any
from paperforge.core.errors import ErrorCode
@dataclass
class PFError:
code: ErrorCode
message: str
details: dict = field(default_factory=dict)
@dataclass
class PFResult:
ok: bool
command: str
version: str
data: Any = None
error: PFError | None = None
def __bool__(self) -> bool:
return self.ok
def to_dict(self) -> dict:
raw: dict[str, Any] = {
"ok": self.ok,
"command": self.command,
"version": self.version,
}
if self.data is not None:
raw["data"] = self.data
else:
raw["data"] = None
if self.error is not None:
raw["error"] = {
"code": self.error.code.value,
"message": self.error.message,
"details": self.error.details,
}
else:
raw["error"] = None
return raw
def to_json(self) -> str:
return json.dumps(self.to_dict(), indent=2, ensure_ascii=False)
@classmethod
def from_dict(cls, data: dict) -> PFResult:
error = None
if data.get("error") is not None:
err_data = data["error"]
error = PFError(
code=ErrorCode(err_data["code"]),
message=err_data["message"],
details=err_data.get("details", {}),
)
return cls(
ok=data["ok"],
command=data["command"],
version=data["version"],
data=data.get("data"),
error=error,
)

105
paperforge/core/state.py Normal file
View file

@ -0,0 +1,105 @@
"""State machine enums and allowed transitions for PaperForge lifecycle."""
from __future__ import annotations
from enum import Enum
from typing import ClassVar
class OcrStatus(str, Enum):
"""OCR processing state for a paper."""
PENDING = "pending"
PROCESSING = "processing"
DONE = "done"
FAILED = "failed"
@classmethod
def from_legacy(cls, value: str) -> OcrStatus:
"""Map legacy OCR state strings to canonical enum."""
mapping = {
"pending": cls.PENDING,
"processing": cls.PROCESSING,
"queued": cls.PENDING,
"running": cls.PROCESSING,
"done": cls.DONE,
"failed": cls.FAILED,
"error": cls.FAILED,
"blocked": cls.FAILED,
"nopdf": cls.FAILED,
"done_incomplete": cls.DONE,
}
return mapping.get(value.strip().lower(), cls.PENDING)
def __str__(self) -> str:
return self.value
class PdfStatus(str, Enum):
"""PDF attachment health state."""
HEALTHY = "healthy"
BROKEN = "broken"
MISSING = "missing"
def __str__(self) -> str:
return self.value
class Lifecycle(str, Enum):
"""Derived lifecycle stage summarizing the paper's overall progress."""
PDF_READY = "pdf_ready"
OCR_READY = "ocr_ready"
ANALYZE_READY = "analyze_ready"
DEEP_READ_DONE = "deep_read_done"
ERROR_STATE = "error_state"
def __str__(self) -> str:
return self.value
class ALLOWED_TRANSITIONS:
"""Legal state migration tables.
Workers MUST validate transitions via check() before writing state changes.
"""
OCR_STATUS: ClassVar[dict[OcrStatus, list[OcrStatus]]] = {
OcrStatus.PENDING: [OcrStatus.PROCESSING, OcrStatus.FAILED],
OcrStatus.PROCESSING: [OcrStatus.DONE, OcrStatus.FAILED],
OcrStatus.DONE: [OcrStatus.PENDING], # re-run
OcrStatus.FAILED: [OcrStatus.PENDING], # retry
}
DEEP_READING_STATUS: ClassVar[dict[str, list[str]]] = {
"pending": ["done"],
"done": ["pending"], # re-run with --force
}
LIFECYCLE: ClassVar[dict[Lifecycle, list[Lifecycle]]] = {
Lifecycle.PDF_READY: [Lifecycle.OCR_READY, Lifecycle.ERROR_STATE],
Lifecycle.OCR_READY: [Lifecycle.ANALYZE_READY, Lifecycle.ERROR_STATE],
Lifecycle.ANALYZE_READY: [Lifecycle.DEEP_READ_DONE, Lifecycle.ERROR_STATE],
Lifecycle.DEEP_READ_DONE: [Lifecycle.PDF_READY, Lifecycle.ERROR_STATE], # re-run
Lifecycle.ERROR_STATE: [Lifecycle.PDF_READY], # recover
}
@staticmethod
def check_ocr(current: OcrStatus, target: OcrStatus) -> tuple[bool, str]:
"""Validate OCR status transition."""
allowed = ALLOWED_TRANSITIONS.OCR_STATUS.get(current, [])
if target in allowed:
return True, ""
return False, (
f"Illegal OCR transition: {current.value} -> {target.value}. "
f"Allowed from '{current.value}': {[s.value for s in allowed]}"
)
@staticmethod
def check_lifecycle(current: Lifecycle, target: Lifecycle) -> tuple[bool, str]:
"""Validate lifecycle transition."""
allowed = ALLOWED_TRANSITIONS.LIFECYCLE.get(current, [])
if target in allowed:
return True, ""
return False, (
f"Illegal lifecycle transition: {current.value} -> {target.value}. "
f"Allowed from '{current.value}': {[s.value for s in allowed]}"
)

View file

View file

@ -0,0 +1,157 @@
"""Field registry validation — used by paperforge doctor to detect state drift."""
from __future__ import annotations
from pathlib import Path
from typing import Any
def validate_entry_fields(
entry: dict,
owner: str,
registry: dict,
entry_label: str = "",
) -> list[dict]:
"""Validate a single entry's fields against the field registry.
Returns list of issues, each with:
- severity: "error" | "warning" | "info"
- code: "MISSING_REQUIRED" | "UNKNOWN_FIELD" | "DRIFT" | "TYPE_MISMATCH"
- message: human-readable description
- field: field name
"""
issues = []
owner_fields = registry.get(owner, {})
if not owner_fields:
return issues
# Check 1: Required fields present
for field_name, meta in owner_fields.items():
if meta.get("required", False) and field_name not in entry:
issues.append({
"severity": "error",
"code": "MISSING_REQUIRED",
"field": field_name,
"message": f"Missing required field '{field_name}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''}",
"suggestion": f"Add '{field_name}' to the entry with appropriate value ({meta.get('type', 'str')})",
})
elif not meta.get("required", True) and field_name not in entry:
issues.append({
"severity": "info",
"code": "MISSING_OPTIONAL",
"field": field_name,
"message": f"Missing optional field '{field_name}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''}",
})
# Check 2: Unknown fields (drift detection)
known_fields = set(owner_fields.keys())
for key in entry:
if key not in known_fields:
issues.append({
"severity": "warning",
"code": "DRIFT",
"field": key,
"message": f"Unknown field '{key}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''} — not in field registry",
"suggestion": f"Either remove '{key}' or add it to field_registry.yaml under '{owner}'",
})
return issues
def validate_collection(
entries: list[dict],
owner: str,
registry: dict,
) -> dict:
"""Validate a collection of entries against field registry.
Returns aggregate diagnostics:
- total_entries: int
- entries_with_errors: int
- entries_with_warnings: int
- issues: list[dict] (all individual issues)
- summary: dict (per-code counts)
- summary_text: str (human-readable)
"""
all_issues = []
error_entries = set()
warning_entries = set()
for i, entry in enumerate(entries):
label = entry.get("zotero_key", f"entry_{i}")
issues = validate_entry_fields(entry, owner, registry, label)
has_error = False
has_warning = False
for issue in issues:
all_issues.append(issue)
if issue["severity"] == "error":
has_error = True
elif issue["severity"] == "warning":
has_warning = True
if has_error:
error_entries.add(label)
if has_warning:
warning_entries.add(label)
# Summary
code_counts: dict[str, int] = {}
for issue in all_issues:
code = issue["code"]
code_counts[code] = code_counts.get(code, 0) + 1
lines = []
if all_issues:
lines.append(f"Found {len(all_issues)} field issue(s) across {len(entries)} {owner} entries:")
lines.append(f" Entries with errors: {len(error_entries)}")
lines.append(f" Entries with warnings: {len(warning_entries)}")
for code, count in sorted(code_counts.items()):
lines.append(f" {code}: {count}")
else:
lines.append(f"All {len(entries)} {owner} entries are valid against field registry.")
# Add per-entry details for actionable items
if all_issues:
lines.append("")
lines.append("Details:")
for issue in all_issues[:20]: # cap display
lines.append(f" [{issue['severity'].upper()}] {issue['message']}")
if issue.get("suggestion"):
lines.append(f" Suggestion: {issue['suggestion']}")
if len(all_issues) > 20:
lines.append(f" ... and {len(all_issues) - 20} more issues")
return {
"total_entries": len(entries),
"entries_with_errors": len(error_entries),
"entries_with_warnings": len(warning_entries),
"issues": all_issues,
"summary": code_counts,
"summary_text": "\n".join(lines),
}
def validate_frontmatter_from_file(
file_path: Path,
registry: dict,
) -> list[dict]:
"""Read a single formal note and validate its frontmatter against registry."""
try:
text = file_path.read_text(encoding="utf-8")
except Exception:
return [{"severity": "error", "code": "READ_ERROR", "field": "", "message": f"Cannot read {file_path}"}]
# Parse frontmatter
import re
fm_match = re.match(r"^---\s*\n(.*?)\n---", text, re.DOTALL)
if not fm_match:
return [{"severity": "warning", "code": "NO_FRONTMATTER", "field": "", "message": f"No frontmatter found in {file_path.name}"}]
frontmatter = {}
for line in fm_match.group(1).splitlines():
line = line.strip()
if ":" in line:
key, _, val = line.partition(":")
frontmatter[key.strip()] = val.strip()
return validate_entry_fields(frontmatter, "frontmatter", registry, file_path.stem)

View file

@ -336,7 +336,6 @@ class PaperForgeStatusView extends ItemView {
}
_fetchStats(quiet) {
// Phase 25: Read canonical index JSON directly (D-05)
if (!quiet && !this._cachedStats) {
this._metricsEl.empty();
this._metricsEl.createEl('div', { cls: 'paperforge-status-loading', text: 'Loading...' });
@ -346,6 +345,47 @@ class PaperForgeStatusView extends ItemView {
const vp = this.app.vault.adapter.basePath;
const plugin = this.app.plugins.plugins['paperforge'];
const { path: pythonExe, extraArgs = [] } = resolvePythonExecutable(vp, plugin?.settings);
// Plan 57-04: Try paperforge dashboard --json first, fallback to file
execFile(pythonExe, [...extraArgs, '-m', 'paperforge', 'dashboard', '--json'], { cwd: vp, timeout: 30000 }, (err, stdout) => {
if (!err) {
try {
const body = JSON.parse(stdout);
if (body.ok && body.data) {
const d = this._normalizeDashboardData(body.data);
this._cachedStats = d;
this._metricsEl.empty();
this._renderStats(d);
this._renderOcr(d);
this._dashboardPermissions = body.data.permissions || {};
return;
}
} catch (_) {
// Fall through to file fallback
}
}
// Fallback: read formal-library.json directly
this._fallbackFetchStats(quiet, vp, plugin);
});
}
_normalizeDashboardData(data) {
const stats = data.stats || {};
const ocrHealth = stats.ocr_health || {};
const pdfHealth = stats.pdf_health || {};
const ocrTotal = (ocrHealth.done || 0) + (ocrHealth.pending || 0) + (ocrHealth.failed || 0);
return {
total_papers: stats.papers || 0,
formal_notes: stats.papers || 0,
exports: 0,
bases: 0,
ocr: { total: ocrTotal, pending: ocrHealth.pending || 0, processing: 0, done: ocrHealth.done || 0, failed: ocrHealth.failed || 0 },
path_errors: (pdfHealth.broken || 0) + (pdfHealth.missing || 0),
};
}
_fallbackFetchStats(quiet, vp, plugin) {
const systemDir = plugin?.settings?.system_dir || 'System';
const indexPath = path.join(vp, systemDir, 'PaperForge', 'indexes', 'formal-library.json');
@ -402,11 +442,10 @@ class PaperForgeStatusView extends ItemView {
this._renderStats(this._cachedStats);
this._renderOcr(this._cachedStats);
} catch (err) {
// D-07: Fallback — spawn CLI if file is missing or corrupt
// D-07: Second fallback — spawn CLI if file is missing or corrupt
if (!quiet && !this._cachedStats) {
this._metricsEl.createEl('div', { cls: 'paperforge-status-loading', text: 'No index \u2014 trying CLI...' });
}
const plugin = this.app.plugins.plugins['paperforge'];
const { path: pythonExe, extraArgs = [] } = resolvePythonExecutable(vp, plugin?.settings);
execFile(pythonExe, [...extraArgs, '-m', 'paperforge', 'status', '--json'], { cwd: vp, timeout: 30000 }, (err2, stdout) => {
if (err2) {
@ -1837,7 +1876,7 @@ class PaperForgeSettingTab extends PluginSettingTab {
_syncRuntime(btn) {
const vp = this.app.vault.adapter.basePath;
const { path: pythonExe, extraArgs = [] } = resolvePythonExecutable(vp, this.plugin.settings);
const ver = this.plugin.manifest.version || '1.4.17rc2';
const ver = this.plugin.manifest.version;
const installCmd = buildRuntimeInstallCommand(pythonExe, ver, extraArgs);
btn.setDisabled(true);
@ -2417,7 +2456,7 @@ class PaperForgeSetupModal extends Modal {
if (!hasPaperforge) {
this._log(t('install_bootstrapping'));
const ver = this.plugin.manifest.version || '1.4.17rc2';
const ver = this.plugin.manifest.version;
await runPython([
'-m', 'pip', 'install', '--upgrade',
`git+https://github.com/LLLin000/PaperForge.git@${ver}`,
@ -2668,7 +2707,7 @@ module.exports = class PaperForgePlugin extends Plugin {
_autoUpdate() {
const vp = this.app.vault.adapter.basePath;
const { path: pythonExe, extraArgs = [] } = resolvePythonExecutable(vp, this.settings);
const ver = this.manifest.version || '1.4.17rc2';
const ver = this.manifest.version;
const url = `git+https://github.com/LLLin000/PaperForge.git@${ver}`;
// Check if installed package version matches plugin version

View file

@ -1,7 +1,7 @@
{
"id": "paperforge",
"name": "PaperForge",
"version": "1.4.17rc3",
"version": "1.4.17rc4",
"minAppVersion": "1.9.0",
"description": "PaperForge — Zotero literature pipeline. Sync PDFs, run OCR, and read with AI-assisted deep reading.",
"author": "Lin Zhaoxuan",

View file

@ -1,4 +1,5 @@
{
"1.4.3": "1.9.0",
"1.4.17rc3": "1.9.0"
"1.4.3": "1.0.0",
"1.4.17rc3": "1.9.0",
"1.4.17rc4": "1.9.0"
}

View file

@ -0,0 +1,35 @@
"""Field registry: schema definitions for all field-carrying structures."""
from __future__ import annotations
from pathlib import Path
def load_field_registry(path: Path | None = None) -> dict:
"""Load field registry from YAML file.
Returns nested dict: {owner: {field: {type, required, public, description}}}
"""
import yaml
if path is None:
path = Path(__file__).resolve().parent / "field_registry.yaml"
if not path.exists():
return {}
with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
return data if isinstance(data, dict) else {}
def get_owner_fields(registry: dict, owner: str) -> dict:
"""Get all field definitions for a given owner."""
return registry.get(owner, {})
def get_field_info(registry: dict, owner: str, field_name: str) -> dict | None:
"""Get metadata for a specific field in an owner."""
owner_fields = get_owner_fields(registry, owner)
return owner_fields.get(field_name)

View file

@ -0,0 +1,215 @@
frontmatter:
zotero_key:
type: str
required: true
public: true
description: "Zotero citation key"
domain:
type: str
required: true
public: true
description: "Classification domain"
title:
type: str
required: true
public: true
description: "Paper title"
year:
type: str
required: false
public: true
description: "Publication year"
doi:
type: str
required: false
public: true
description: "Digital Object Identifier"
collection_path:
type: str
required: false
public: true
description: "Zotero collection path"
has_pdf:
type: bool
required: true
public: true
description: "Has PDF attachment"
pdf_path:
type: str
required: false
public: true
description: "Wikilink to main PDF"
supplementary:
type: list
required: false
public: true
description: "Wikilinks to supplementary PDFs"
fulltext_md_path:
type: str
required: false
public: true
description: "Wikilink to OCR fulltext"
recommend_analyze:
type: bool
required: false
public: true
description: "System recommends analysis"
analyze:
type: bool
required: false
public: true
description: "User requests deep reading"
do_ocr:
type: bool
required: false
public: true
description: "User requests OCR"
ocr_status:
type: str
required: false
public: true
description: "OCR processing status"
deep_reading_status:
type: str
required: false
public: true
description: "Deep reading status"
path_error:
type: str
required: false
public: true
description: "Path resolution error"
index_entry:
file:
type: str
required: true
public: true
description: "Path to formal note"
zotero_key:
type: str
required: true
public: true
description: "Zotero citation key"
domain:
type: str
required: true
public: true
description: "Classification domain"
title:
type: str
required: true
public: true
description: "Paper title"
year:
type: str
required: false
public: true
description: "Publication year"
doi:
type: str
required: false
public: true
description: "DOI"
collection_path:
type: str
required: false
public: true
description: "Zotero collection path"
has_pdf:
type: bool
required: true
public: true
description: "Has PDF"
pdf_path:
type: str
required: false
public: true
description: "PDF path"
supplementary:
type: list
required: false
public: true
description: "Supplementary PDFs"
fulltext_md_path:
type: str
required: false
public: true
description: "Fulltext path"
recommend_analyze:
type: bool
required: false
public: true
description: "Recommend analysis"
analyze:
type: bool
required: false
public: true
description: "Analysis flag"
do_ocr:
type: bool
required: false
public: true
description: "OCR flag"
ocr_status:
type: str
required: false
public: true
description: "OCR status"
deep_reading_status:
type: str
required: false
public: true
description: "Deep reading status"
path_error:
type: str
required: false
public: true
description: "Path error"
lifecycle:
type: str
required: false
public: true
description: "Derived lifecycle stage"
health:
type: str
required: false
public: true
description: "Derived health dimension scores"
maturity:
type: str
required: false
public: true
description: "Derived maturity score"
next_step:
type: str
required: false
public: true
description: "Recommended next action"
ocr_meta:
ocr_status:
type: str
required: true
public: true
description: "OCR processing status"
error:
type: str
required: false
public: true
description: "Error message if failed"
start_time:
type: str
required: false
public: false
description: "OCR start timestamp"
completion_time:
type: str
required: false
public: false
description: "OCR completion timestamp"
page_count:
type: int
required: false
public: false
description: "Number of pages processed"

View file

View file

@ -0,0 +1,78 @@
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any
from paperforge.adapters.bbt import (
extract_authors,
_identify_main_pdf,
_normalize_attachment_path,
load_export_rows,
resolve_item_collection_paths,
)
from paperforge.adapters.obsidian_frontmatter import (
candidate_markdown,
read_frontmatter_dict,
update_frontmatter_field,
)
from paperforge.adapters.zotero_paths import (
absolutize_vault_path,
obsidian_wikilink_for_path,
obsidian_wikilink_for_pdf,
)
logger = logging.getLogger(__name__)
class SyncService:
"""Orchestrates the sync lifecycle: load BBT exports, match entries, generate notes."""
def __init__(self, vault: Path):
self.vault = vault
self.paths: dict[str, Path] = {}
self._resolved = False
def resolve_paths(self) -> dict[str, Path]:
"""Resolve vault paths using pipeline_paths."""
from paperforge.worker._utils import pipeline_paths
self.paths = pipeline_paths(self.vault)
self._resolved = True
return self.paths
def load_exports(self) -> list[dict]:
"""Load all BBT export JSON files."""
from paperforge.config import load_vault_config
cfg = load_vault_config(self.vault)
system_dir = self.vault / cfg.get("system_dir", "99_System")
exports_dir = system_dir / "PaperForge" / "exports"
if not exports_dir.exists():
logger.warning("Exports directory not found: %s", exports_dir)
return []
rows = []
for f in sorted(exports_dir.glob("*.json")):
try:
rows.extend(load_export_rows(f))
except Exception as e:
logger.error("Failed to load %s: %s", f.name, e)
return rows
def build_candidates(self, rows: list[dict]) -> list[dict]:
"""Generate candidate markdown entries from BBT rows."""
candidates = []
for row in rows:
try:
markdown = candidate_markdown(row)
candidates.append({"row": row, "markdown": markdown})
except Exception as e:
logger.error("Failed to build candidate: %s", e)
return candidates
def run_sync(self, verbose: bool = False, json_output: bool = False) -> int | dict:
"""Full sync orchestration. Delegates to run_selection_sync logic.
Returns int (exit code) for CLI mode, dict for json_output mode.
"""
from paperforge.worker.sync import run_selection_sync
return run_selection_sync(self.vault, verbose=verbose, json_output=json_output)

View file

@ -0,0 +1,24 @@
"""PaperForge setup package — modular setup components."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
@dataclass
class SetupStepResult:
"""Result of a single setup step."""
step: str
ok: bool = True
message: str = ""
error: str | None = None
details: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"step": self.step,
"ok": self.ok,
"message": self.message,
"error": self.error,
}

133
paperforge/setup/agent.py Normal file
View file

@ -0,0 +1,133 @@
"""AgentInstaller — deploys skill files and agent configs."""
from __future__ import annotations
import shutil
from pathlib import Path
from paperforge.setup import SetupStepResult
class AgentInstaller:
"""Deploy agent skill files, command files, and rules."""
def __init__(self, vault: Path, agent_type: str = "opencode"):
self.vault = vault
self.agent_type = agent_type
self._script_dir = Path(__file__).resolve().parent.parent
def _get_skills_dir(self) -> Path:
"""Get the target skills directory based on agent type."""
if self.agent_type == "opencode":
base = Path.home() / ".config" / "opencode"
elif self.agent_type == "claude":
base = Path.home() / ".claude"
elif self.agent_type == "codex":
base = Path.home() / ".codex"
else:
base = self.vault / ".agents"
return base / "skills"
def deploy_skills(self) -> SetupStepResult:
"""Deploy literature-qa skill directory to agent config."""
source_skills = self._script_dir / "skills" / "literature-qa"
if not source_skills.exists():
return SetupStepResult(
step="agent_installer",
ok=False,
message="Skill source directory not found",
error=f"Not found: {source_skills}",
)
target_dir = self._get_skills_dir() / "literature-qa"
target_dir.mkdir(parents=True, exist_ok=True)
try:
if source_skills.is_dir():
shutil.copytree(source_skills, target_dir, dirs_exist_ok=True)
else:
shutil.copy2(source_skills, target_dir)
return SetupStepResult(
step="agent_installer",
ok=True,
message=f"Deployed literature-qa skill to {target_dir}",
details={"source": str(source_skills), "target": str(target_dir)},
)
except Exception as e:
return SetupStepResult(
step="agent_installer",
ok=False,
message="Failed to deploy skills",
error=str(e),
)
def deploy_commands(self) -> SetupStepResult:
"""Deploy command files to vault agent config dir."""
source_commands = self._script_dir / "command_files"
if not source_commands.exists():
return SetupStepResult(
step="agent_installer",
ok=True,
message="No command files to deploy",
details={"skipped": True},
)
agent_dir = self.vault / ".agents"
target_dir = agent_dir / "command_files"
target_dir.mkdir(parents=True, exist_ok=True)
try:
for f in source_commands.iterdir():
if f.is_file():
shutil.copy2(f, target_dir / f.name)
return SetupStepResult(
step="agent_installer",
ok=True,
message=f"Deployed command files to {target_dir}",
details={"source": str(source_commands), "target": str(target_dir)},
)
except Exception as e:
return SetupStepResult(
step="agent_installer",
ok=False,
message="Failed to deploy command files",
error=str(e),
)
def deploy_agent_config(self) -> SetupStepResult:
"""Deploy AGENTS.md and other agent config files."""
source_agents = self._script_dir.parent / "AGENTS.md"
if not source_agents.exists():
return SetupStepResult(
step="agent_installer",
ok=True,
message="No AGENTS.md to deploy",
details={"skipped": True},
)
try:
target = self.vault / "AGENTS.md"
shutil.copy2(source_agents, target)
return SetupStepResult(
step="agent_installer",
ok=True,
message=f"Deployed AGENTS.md to {target}",
details={"source": str(source_agents), "target": str(target)},
)
except Exception as e:
return SetupStepResult(
step="agent_installer",
ok=False,
message="Failed to deploy AGENTS.md",
error=str(e),
)
def run_all(self) -> list[SetupStepResult]:
"""Run all deployment steps."""
results = []
results.append(self.deploy_skills())
results.append(self.deploy_commands())
results.append(self.deploy_agent_config())
return results

View file

@ -0,0 +1,68 @@
"""SetupChecker — validates preconditions before setup begins."""
from __future__ import annotations
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Any
from paperforge.setup import SetupStepResult
class SetupChecker:
"""Validate all preconditions before any installation step."""
MIN_PYTHON = (3, 10)
def __init__(self, vault: Path):
self.vault = vault
def run(self) -> SetupStepResult:
"""Run all precondition checks. Returns result with details."""
issues: list[str] = []
details: dict[str, Any] = {}
# Check Python version
py_version = sys.version_info[:2]
details["python_version"] = f"{py_version[0]}.{py_version[1]}"
if py_version < self.MIN_PYTHON:
issues.append(f"Python {py_version[0]}.{py_version[1]} < {self.MIN_PYTHON[0]}.{self.MIN_PYTHON[1]}")
# Check pip availability
pip_path = shutil.which("pip") or shutil.which("pip3")
details["pip_found"] = pip_path is not None
if not pip_path:
issues.append("pip not found in PATH")
# Check vault directory
details["vault_exists"] = self.vault.exists()
if not self.vault.exists():
issues.append(f"Vault directory not found: {self.vault}")
# Check Zotero
zotero_path = shutil.which("zotero") or shutil.which("zotero.exe")
details["zotero_found"] = zotero_path is not None
# Check Better BibTeX exports
system_dir = self.vault / "99_System"
exports_dir = system_dir / "PaperForge" / "exports"
bbt_exists = exports_dir.exists() and len(list(exports_dir.glob("*.json"))) > 0
details["bbt_exports_found"] = bbt_exists
if issues:
return SetupStepResult(
step="checker",
ok=False,
message=f"Precondition checks failed ({len(issues)} issue(s))",
error="; ".join(issues),
details=details,
)
return SetupStepResult(
step="checker",
ok=True,
message="All preconditions met",
details=details,
)

View file

@ -0,0 +1,80 @@
"""ConfigWriter — atomic paperforge.json writer."""
from __future__ import annotations
import json
import os
import tempfile
from pathlib import Path
from paperforge.setup import SetupStepResult
class ConfigWriter:
"""Write paperforge.json atomically using tempfile + os.replace."""
REQUIRED_KEYS = ["system_dir", "resources_dir", "literature_dir", "control_dir"]
def __init__(self, vault: Path):
self.vault = vault
self.config_path = vault / "paperforge.json"
def write(self, config: dict) -> SetupStepResult:
"""Write paperforge.json atomically."""
# Validate required keys
missing = [k for k in self.REQUIRED_KEYS if k not in config]
if missing:
return SetupStepResult(
step="config_writer",
ok=False,
message="Missing required config keys",
error=f"Missing: {', '.join(missing)}",
details={"missing_keys": missing},
)
try:
# Write to temp file, then atomic replace
fd, tmp_path = tempfile.mkstemp(
suffix=".json",
prefix="paperforge_",
dir=str(self.vault),
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, str(self.config_path))
except Exception:
# Clean up temp file on failure
try:
os.unlink(tmp_path)
except Exception:
pass
raise
return SetupStepResult(
step="config_writer",
ok=True,
message=f"paperforge.json written to {self.config_path}",
details={"path": str(self.config_path), "keys": list(config.keys())},
)
except Exception as e:
return SetupStepResult(
step="config_writer",
ok=False,
message="Failed to write config",
error=str(e),
)
def read(self) -> dict | None:
"""Read existing config, return None if not exists."""
if not self.config_path.exists():
return None
try:
with open(self.config_path, "r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return None
def exists(self) -> bool:
"""Check if config file exists."""
return self.config_path.exists()

97
paperforge/setup/plan.py Normal file
View file

@ -0,0 +1,97 @@
"""SetupPlan — orchestrates all setup steps in sequence."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Callable
from paperforge.setup import SetupStepResult
from paperforge.setup.checker import SetupChecker
from paperforge.setup.config_writer import ConfigWriter
from paperforge.setup.vault import VaultInitializer
from paperforge.setup.runtime import RuntimeInstaller
from paperforge.setup.agent import AgentInstaller
ProgressCallback = Callable[[str], None]
class SetupPlan:
"""Orchestrate the setup lifecycle: check -> config -> vault -> install -> agent."""
def __init__(
self,
vault: Path,
config: dict | None = None,
env_values: dict[str, str] | None = None,
zotero_path: str | None = None,
agent_type: str = "opencode",
version: str | None = None,
progress_callback: ProgressCallback | None = None,
):
self.vault = vault
self.config = config or {}
self.env_values = env_values or {}
self.zotero_path = zotero_path
self.agent_type = agent_type
self.version = version
self.progress_callback = progress_callback
def _log(self, message: str) -> None:
if self.progress_callback:
self.progress_callback(message)
def execute(self, json_output: bool = False) -> list[SetupStepResult] | int:
"""Run all setup steps in sequence.
Args:
json_output: If True, return results list as JSON string.
If False, return exit code (0 = success, 1 = failure).
"""
results: list[SetupStepResult] = []
# Step 1: Checker
self._log("Checking preconditions...")
checker = SetupChecker(self.vault)
results.append(checker.run())
# Step 2: Config writer
self._log("Writing config...")
writer = ConfigWriter(self.vault)
results.append(writer.write(self.config))
# Step 3: Vault initializer
self._log("Initializing vault structure...")
vault_init = VaultInitializer(self.vault, self.config)
results.append(vault_init.create_directories())
results.append(vault_init.create_zotero_junction(self.zotero_path))
results.append(vault_init.merge_env(self.env_values))
# Step 4: Runtime installer
self._log("Installing runtime...")
installer = RuntimeInstaller(self.vault, version=self.version, progress_callback=self._log)
results.append(installer.install())
# Step 5: Agent installer
self._log("Deploying agent config...")
agent = AgentInstaller(self.vault, agent_type=self.agent_type)
agent_results = agent.run_all()
results.extend(agent_results)
if json_output:
output = [r.to_dict() for r in results]
print(json.dumps(output, indent=2, ensure_ascii=False))
return 0
# Print summary
ok_count = sum(1 for r in results if r.ok)
total = len(results)
print(f"Setup complete: {ok_count}/{total} steps passed")
for r in results:
status = "PASS" if r.ok else "FAIL"
print(f" [{status}] {r.step}: {r.message}")
if not r.ok and r.error:
print(f" Error: {r.error}")
return 0 if all(r.ok for r in results) else 1

103
paperforge/setup/runtime.py Normal file
View file

@ -0,0 +1,103 @@
"""RuntimeInstaller -- pip install with version pinning."""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
from typing import Callable
from paperforge.core.errors import ErrorCode
from paperforge.setup import SetupStepResult
ProgressCallback = Callable[[str], None]
class RuntimeInstaller:
"""Install PaperForge Python package with version pinning."""
def __init__(
self,
vault: Path,
version: str | None = None,
progress_callback: ProgressCallback | None = None,
):
self.vault = vault
self.version = version
self.progress_callback = progress_callback
def _log(self, message: str) -> None:
if self.progress_callback:
self.progress_callback(message)
def _pip_install(
self, package_spec: str
) -> tuple[bool, str, str]:
"""Run pip install and return (ok, stdout, stderr)."""
try:
result = subprocess.run(
[sys.executable, "-m", "pip", "install", package_spec],
capture_output=True,
text=True,
timeout=120,
)
return (result.returncode == 0, result.stdout, result.stderr)
except subprocess.TimeoutExpired:
return (False, "", "pip install timed out after 120s")
except Exception as e:
return (False, "", str(e))
def install(self) -> SetupStepResult:
"""Install paperforge via pip with optional version pin."""
self._log("Installing PaperForge...")
if self.version:
package_spec = (
f"git+https://github.com/LLLin000/PaperForge.git@{self.version}"
)
else:
package_spec = "git+https://github.com/LLLin000/PaperForge.git"
ok, stdout, stderr = self._pip_install(package_spec)
if ok:
return SetupStepResult(
step="runtime_installer",
ok=True,
message=(
f"PaperForge installed successfully{(' (' + self.version + ')') if self.version else ''}"
),
details={"version": self.version or "latest", "stdout": stdout[:500]},
)
else:
error_code = ErrorCode.INTERNAL_ERROR
if "pip" in stderr.lower():
error_code = ErrorCode.INTERNAL_ERROR
elif "connection" in stderr.lower() or "timeout" in stderr.lower():
error_code = ErrorCode.INTERNAL_ERROR
return SetupStepResult(
step="runtime_installer",
ok=False,
message="Failed to install PaperForge",
error=f"[{error_code.value}] {stderr[:500]}",
details={"version": self.version or "latest", "stderr": stderr[:500]},
)
def check_installed(self) -> bool:
"""Check if paperforge is already installed."""
try:
result = subprocess.run(
[
sys.executable,
"-c",
"from paperforge import __version__; print(__version__)",
],
capture_output=True,
text=True,
timeout=10,
)
return result.returncode == 0 and bool(result.stdout.strip())
except Exception:
return False

150
paperforge/setup/vault.py Normal file
View file

@ -0,0 +1,150 @@
"""VaultInitializer -- creates vault directory structure and env config."""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
from paperforge.setup import SetupStepResult
class VaultInitializer:
"""Create vault directory structure, Zotero junction, and .env file."""
DEFAULT_DIRS = {
"system_dir": "99_System",
"resources_dir": "03_Resources",
"literature_dir": "Literature",
"control_dir": "99_System/PaperForge",
}
def __init__(self, vault: Path, config: dict):
self.vault = vault
self.config = config
def create_directories(self) -> SetupStepResult:
"""Create all required vault directories."""
dirs_to_create = [
self.vault / "paperforge.json",
]
for key in ("system_dir", "resources_dir", "literature_dir", "control_dir"):
rel = self.config.get(key, self.DEFAULT_DIRS.get(key, ""))
if rel:
dirs_to_create.append(self.vault / rel)
created = []
existing = []
for d in dirs_to_create:
if d.suffix:
d.parent.mkdir(parents=True, exist_ok=True)
continue
if not d.exists():
d.mkdir(parents=True, exist_ok=True)
created.append(str(d.relative_to(self.vault)))
else:
existing.append(str(d.relative_to(self.vault)))
return SetupStepResult(
step="vault_initializer",
ok=True,
message=f"Created {len(created)} director(ies), {len(existing)} already exist",
details={"created": created, "existing": existing},
)
def create_zotero_junction(self, zotero_path: str | None = None) -> SetupStepResult:
"""Create Zotero junction/symlink to vault."""
system_dir = self.vault / self.config.get(
"system_dir", self.DEFAULT_DIRS["system_dir"]
)
zotero_link = system_dir / "Zotero"
if zotero_link.exists() or zotero_link.is_symlink():
return SetupStepResult(
step="vault_initializer",
ok=True,
message=f"Zotero link already exists at {zotero_link}",
details={"path": str(zotero_link)},
)
if not zotero_path:
return SetupStepResult(
step="vault_initializer",
ok=True,
message="Zotero path not provided -- skipping junction creation",
details={"skipped": True},
)
try:
if os.name == "nt":
result = subprocess.run(
["cmd", "/c", "mklink", "/J", str(zotero_link), zotero_path],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0:
return SetupStepResult(
step="vault_initializer",
ok=False,
message="Failed to create Zotero junction",
error=result.stderr.strip() or result.stdout.strip(),
)
else:
zotero_link.symlink_to(zotero_path, target_is_directory=True)
return SetupStepResult(
step="vault_initializer",
ok=True,
message=f"Zotero junction created: {zotero_link} -> {zotero_path}",
details={"source": str(zotero_link), "target": zotero_path},
)
except Exception as e:
return SetupStepResult(
step="vault_initializer",
ok=False,
message="Failed to create Zotero junction",
error=str(e),
)
def merge_env(self, env_values: dict[str, str]) -> SetupStepResult:
"""Merge values into .env file."""
env_path = self.vault / ".env"
existing = {}
if env_path.exists():
try:
for line in env_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, _, val = line.partition("=")
existing[key.strip()] = val.strip()
except Exception:
pass
updated = dict(existing)
updated.update(env_values)
try:
lines = [f"{k}={v}\n" for k, v in updated.items()]
env_path.write_text("".join(lines), encoding="utf-8")
added = [k for k in env_values if k not in existing]
return SetupStepResult(
step="vault_initializer",
ok=True,
message=f"Created .env with {len(updated)} entries ({len(added)} new)",
details={
"path": str(env_path),
"total_keys": len(updated),
"new_keys": added,
},
)
except Exception as e:
return SetupStepResult(
step="vault_initializer",
ok=False,
message="Failed to write .env",
error=str(e),
)

View file

@ -23,6 +23,10 @@ from pathlib import Path
from paperforge import __version__
# Backward-compat imports (v2.1 modular setup)
from paperforge.setup.checker import SetupChecker
from paperforge.setup.config_writer import ConfigWriter
if sys.platform == "win32":
import winreg
else:

View file

@ -396,7 +396,7 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
}
# ---- derived state (Phase 24: lifecycle, health, maturity, next-step) ----
entry["lifecycle"] = compute_lifecycle(entry)
entry["lifecycle"] = str(compute_lifecycle(entry))
entry["health"] = compute_health(entry)
entry["maturity"] = compute_maturity(entry)
entry["next_step"] = compute_next_step(entry)

View file

@ -13,6 +13,8 @@ Exports:
from __future__ import annotations
from paperforge.core.state import Lifecycle
def compute_lifecycle(entry: dict) -> str:
"""Derive the current lifecycle state from an index entry.
@ -23,6 +25,10 @@ def compute_lifecycle(entry: dict) -> str:
- **fulltext_ready**: ocr_status == "done" (validated), deep_reading NOT done
- **deep_read_done**: OCR done AND deep-reading done
Returns ``Lifecycle`` enum members where an equivalent exists, or plain
strings for states not yet represented in the enum (``"indexed"``,
``"fulltext_ready"``).
Uses .get() with safe defaults for all field accesses.
"""
has_pdf = bool(entry.get("has_pdf", False))
@ -36,11 +42,11 @@ def compute_lifecycle(entry: dict) -> str:
# OCR validated done opens the door to fulltext_ready and beyond
if ocr_status == "done":
if deep_reading_status == "done":
return "deep_read_done"
return Lifecycle.DEEP_READ_DONE
return "fulltext_ready"
# pdf_ready: has PDF attachment but OCR not validated done
return "pdf_ready"
return Lifecycle.PDF_READY
def compute_health(entry: dict) -> dict[str, str]:

View file

@ -24,6 +24,8 @@ from paperforge.worker._utils import (
)
from paperforge.worker.base_views import ensure_base_views
from paperforge.core.result import PFResult
logger = logging.getLogger(__name__)
@ -356,7 +358,7 @@ def _query_resolved_module(interp: str, extra_args: list[str], module_name: str)
return None
def run_doctor(vault: Path, verbose: bool = False) -> int:
def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> int:
"""Validate PaperForge setup and report by category.
Returns:
@ -445,12 +447,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
ver = mod_info_resolved.get("version")
ver_str = f" ({ver})" if ver else ""
# PyYAML specific: warn if version < 6.0
# PyYAML specific: fail if version < 6.0 (hard dependency in pyproject.toml)
if mod_name == "yaml" and ver:
try:
ver_parts = str(ver).split(".")
if int(ver_parts[0]) < 6:
add_check("Python 环境", "warn",
add_check("Python 环境", "fail",
f"{mod_info['label']} {ver} (需要 >=6.0)",
f"运行: {interp} -m pip install {mod_info['pip']}")
continue
@ -589,6 +591,34 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
check_pdf_paths(vault, paths, add_check)
check_wikilink_format(vault, paths, add_check)
# --- Field registry validation (Phase 59) ---
try:
from paperforge.schema import load_field_registry
from paperforge.doctor.field_validator import validate_frontmatter_from_file
registry = load_field_registry()
except Exception:
registry = {}
if registry:
literature_dir = paths.get("literature")
if literature_dir and literature_dir.exists():
note_files = sorted(literature_dir.rglob("*.md"))
total_issues = 0
for note_file in note_files:
if note_file.name in ("fulltext.md", "deep-reading.md", "discussion.md"):
continue
issues = validate_frontmatter_from_file(note_file, registry)
for issue in issues:
total_issues += 1
add_check(
"字段注册表",
"fail" if issue["severity"] == "error" else "warn" if issue["severity"] == "warning" else "pass",
issue["message"],
issue.get("suggestion", ""),
)
if total_issues == 0:
add_check("字段注册表", "pass", "所有 formal note frontmatter 与字段注册表一致")
ld_deep_script = paths.get("ld_deep_script")
skill_dir = None
if ld_deep_script:
@ -760,6 +790,58 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
else:
add_check("Index Health", "info", "No canonical index -- run `paperforge sync` to generate")
if json_output:
checklist_data = [
{"category": cat, "status": st, "message": msg, "fix": fx}
for cat, st, msg, fx in checks
]
status_counts: dict[str, int] = {}
for _, st, _, _ in checks:
status_counts[st] = status_counts.get(st, 0) + 1
_has_fail = any(status == "fail" for _, status, _, _ in checks)
_has_warn = any(status == "warn" for _, status, _, _ in checks)
_verdict = "FAIL" if _has_fail else ("WARN" if _has_warn else "OK")
_fail_categories = set()
for cat, status, _, fx in checks:
if status == "fail" and fx:
_fail_categories.add(cat)
if "PaperForge 包" in _fail_categories and not bool(_fail_categories - {"PaperForge 包"}):
_next_action = "Recommended: run pip install via the resolved interpreter"
elif "Python 环境" in _fail_categories or "Python 环境 (插件)" in _fail_categories:
_next_action = "Recommended: install/upgrade Python to 3.10+"
elif "Vault 结构" in _fail_categories or "Config Migration" in _fail_categories:
_next_action = "Recommended: run `paperforge sync`"
elif "OCR 配置" in _fail_categories:
_next_action = "Recommended: configure PADDLEOCR_API_TOKEN in .env"
elif "BBT 导出" in _fail_categories:
_next_action = "Recommended: configure Better BibTeX auto-export"
elif "Zotero 链接" in _fail_categories or "Path Resolution" in _fail_categories:
_next_action = "Recommended: create Zotero junction"
elif _has_warn:
_next_action = "Recommended: `paperforge doctor` indicates warnings - review above"
else:
_next_action = "Recommended: `paperforge sync` to ensure index is current"
payload = {
"checks": checklist_data,
"summary": {
"total": len(checks),
"pass": status_counts.get("pass", 0),
"warn": status_counts.get("warn", 0),
"fail": status_counts.get("fail", 0),
"info": status_counts.get("info", 0),
},
"verdict": _verdict,
"next_action": _next_action,
}
result = PFResult(
ok=not _has_fail,
command="doctor",
version=__import__("paperforge").__version__,
data=payload,
)
print(result.to_json())
return 0 if not _has_fail else 1
print("PaperForge Doctor")
print("=" * 40)
current_category = ""
@ -971,7 +1053,7 @@ def run_status(vault: Path, verbose: bool = False, json_output: bool = False) ->
maturity_distribution = summary["maturity_distribution"]
if json_output:
data = {
payload = {
"version": __import__("paperforge").__version__,
"vault": str(vault),
"system_dir": cfg["system_dir"],
@ -995,7 +1077,13 @@ def run_status(vault: Path, verbose: bool = False, json_output: bool = False) ->
"health_aggregate": health_aggregate,
"maturity_distribution": maturity_distribution,
}
print(_json.dumps(data, indent=2, ensure_ascii=False))
result = PFResult(
ok=True,
command="status",
version=__import__("paperforge").__version__,
data=payload,
)
print(result.to_json())
return 0
print("PaperForge status")

View file

@ -14,6 +14,25 @@ import requests
from paperforge.config import load_vault_config, paperforge_paths
from paperforge.worker._domain import build_collection_lookup, load_domain_config, load_domain_collections
from paperforge.adapters.zotero_paths import (
absolutize_vault_path,
obsidian_wikilink_for_path,
obsidian_wikilink_for_pdf,
)
from paperforge.adapters.obsidian_frontmatter import (
_add_missing_frontmatter_fields,
_extract_section,
_legacy_control_flags,
_read_frontmatter_bool_from_text,
_read_frontmatter_optional_bool_from_text,
canonicalize_decision,
candidate_markdown,
compute_final_collection,
extract_preserved_deep_reading,
generate_review,
has_deep_reading_content,
update_frontmatter_field,
)
from paperforge.worker._utils import (
_extract_year,
lookup_impact_factor,
@ -28,40 +47,20 @@ from paperforge.worker._utils import (
yaml_quote,
)
import paperforge.worker.asset_index as asset_index
from paperforge.adapters.bbt import (
_identify_main_pdf,
_normalize_attachment_path,
collection_fields,
extract_authors,
load_export_rows,
resolve_item_collection_paths,
)
import paperforge.worker.asset_index as asset_index
logger = logging.getLogger(__name__)
def _read_frontmatter_bool_from_text(text: str, key: str, default: bool = False) -> bool:
match = re.search(rf"^{re.escape(key)}:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE)
if not match:
return default
return match.group(1).lower() == "true"
def _read_frontmatter_optional_bool_from_text(text: str, key: str) -> Optional[bool]:
match = re.search(rf"^{re.escape(key)}:\s*(?:[\"'])?(true|false)(?:[\"'])?\s*$", text, re.MULTILINE | re.IGNORECASE)
if not match:
return None
return match.group(1).lower() == "true"
def _legacy_control_flags(paths: dict[str, Path], zotero_key: str) -> dict[str, Optional[bool]]:
records_root = paths.get("library_records")
if not records_root or not records_root.exists():
return {"do_ocr": None, "analyze": None}
for record_path in records_root.rglob(f"{zotero_key}.md"):
try:
text = record_path.read_text(encoding="utf-8")
except Exception:
continue
return {
"do_ocr": _read_frontmatter_optional_bool_from_text(text, "do_ocr"),
"analyze": _read_frontmatter_optional_bool_from_text(text, "analyze"),
}
return {"do_ocr": None, "analyze": None}
def load_export_inventory(paths: dict[str, Path]) -> dict[str, dict]:
inventory = {"doi": {}, "pmid": {}, "title": {}}
@ -180,533 +179,6 @@ def apply_existing_library_match(row: dict, inventory: dict[str, dict]) -> dict:
return resolved
def resolve_item_collection_paths(item: dict, collection_lookup: dict) -> list[str]:
paths = []
collection_keys = item.get("collections") or []
if collection_keys:
for key in collection_keys:
paths.append(collection_lookup.get("path_by_key", {}).get(key, key))
item_id = item.get("itemID")
if item_id is not None:
paths.extend(collection_lookup.get("paths_by_item_id", {}).get(item_id, []))
return sorted({path for path in paths if path}, key=lambda value: (-value.count("/"), value))
def obsidian_wikilink_for_pdf(pdf_path: str, vault_dir: Path, zotero_dir: Path | None = None) -> str:
text = str(pdf_path or "").strip()
if not text:
return ""
# Handle storage: prefix paths by resolving through zotero_dir
if text.startswith("storage:") and zotero_dir is not None:
storage_rel = text[len("storage:") :].lstrip("/").lstrip("\\")
absolute_pdf_path = zotero_dir / "storage" / storage_rel.replace("/", os.sep)
absolute_str = str(absolute_pdf_path)
else:
absolute_str = absolutize_vault_path(vault_dir, text, resolve_junction=True)
if not absolute_str:
return ""
absolute_path = Path(absolute_str)
try:
relative = absolute_path.relative_to(vault_dir)
except ValueError:
# Path outside vault — try to route through Zotero junction inside vault
if zotero_dir is not None and zotero_dir.exists():
try:
from paperforge.pdf_resolver import resolve_junction
real_zotero = resolve_junction(zotero_dir)
if real_zotero != zotero_dir:
rel_to_zotero = absolute_path.relative_to(real_zotero)
via_junction = zotero_dir / rel_to_zotero
relative = via_junction.relative_to(vault_dir)
return f"[[{relative.as_posix()}]]"
except (ValueError, OSError):
pass
return f"[[{absolute_path.as_posix()}]]"
return f"[[{relative.as_posix()}]]"
def absolutize_vault_path(vault: Path, path: str, resolve_junction: bool = False) -> str:
text = str(path or "").strip()
if not text:
return ""
candidate = Path(text)
result = str(candidate) if candidate.is_absolute() else str((vault / text.replace("/", os.sep)).resolve())
if resolve_junction:
from paperforge.pdf_resolver import resolve_junction
result = str(resolve_junction(Path(result)))
return result
def obsidian_wikilink_for_path(vault: Path, path: str) -> str:
absolute = absolutize_vault_path(vault, path, resolve_junction=True)
if not absolute:
return ""
absolute_path = Path(absolute)
try:
relative = absolute_path.relative_to(vault)
except ValueError:
return f"[[{absolute_path.as_posix()}]]"
return f"[[{relative.as_posix()}]]"
def collection_fields(collection_paths: list[str]) -> dict[str, str | list[str]]:
paths = [path for path in collection_paths if path]
primary = paths[0] if paths else ""
if paths:
primary = sorted(paths, key=lambda value: (value.count("/"), len(value), value), reverse=True)[0]
tags = []
seen = set()
for path in paths:
for part in [segment.strip() for segment in path.split("/") if segment.strip()]:
if part not in seen:
seen.add(part)
tags.append(part)
group = primary
return {"collections": paths, "collection_tags": tags, "collection_group": [group] if group else []}
def extract_authors(item: dict) -> list[str]:
authors = []
for creator in item.get("creators", []):
if creator.get("creatorType") != "author":
continue
full_name = " ".join(
part for part in [creator.get("firstName", ""), creator.get("lastName", "")] if part
).strip()
if full_name:
authors.append(full_name)
elif creator.get("name"):
authors.append(creator["name"])
return authors
def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tuple[str, str, str]:
"""Normalize a BBT attachment path to a consistent storage: format.
Handles three real-world BBT export formats:
1. Absolute Windows paths: D:\\...\\Zotero\\storage\\8CHARKEY\\filename.pdf
-> storage:8CHARKEY/filename.pdf
2. storage: prefix: storage:KEY/filename.pdf -> pass through
3. Bare relative: KEY/filename.pdf -> storage:KEY/filename.pdf
Args:
path: Raw path from BBT JSON attachment.
zotero_dir: Optional absolute path to Zotero data directory for
validating absolute paths.
Returns:
Tuple of (normalized_path, bbt_path_raw, zotero_storage_key).
normalized_path uses forward slashes and storage: prefix for
Zotero storage paths. bbt_path_raw preserves the original input
for debugging. zotero_storage_key is the 8-character Zotero key.
"""
raw = str(path or "").strip()
if not raw:
return ("", "", "")
bbt_path_raw = raw
# Format 2: Already has storage: prefix — pass through with slash normalization
if raw.startswith("storage:"):
storage_rel = raw[len("storage:") :].lstrip("/").lstrip("\\")
storage_rel = storage_rel.replace("\\", "/")
parts = storage_rel.split("/")
zotero_storage_key = parts[0] if parts else ""
return (f"storage:{storage_rel}", bbt_path_raw, zotero_storage_key)
# Format 1: Absolute Windows path pointing to Zotero storage
candidate = Path(raw)
if candidate.is_absolute():
norm_path = raw.replace("\\", "/")
# Detect Zotero storage pattern: .../storage/8CHARKEY/...
if "/storage/" in norm_path:
parts_after_storage = norm_path.split("/storage/", 1)[1]
parts = parts_after_storage.split("/")
if len(parts) >= 2 and len(parts[0]) == 8 and parts[0].isalnum():
zotero_storage_key = parts[0]
filename = "/".join(parts[1:])
return (f"storage:{zotero_storage_key}/{filename}", bbt_path_raw, zotero_storage_key)
# Absolute path but not in Zotero storage — mark as absolute
return (f"absolute:{raw}", bbt_path_raw, "")
# Format 3: Bare relative path — prepend storage: prefix
norm = raw.replace("\\", "/")
parts = norm.split("/")
zotero_storage_key = parts[0] if parts else ""
return (f"storage:{norm}", bbt_path_raw, zotero_storage_key)
def _identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict]]:
"""Identify the main PDF and supplementary materials from attachments.
Uses a hybrid three-priority strategy (Decision D-02):
1. Primary: attachment.title == "PDF" AND contentType == "application/pdf"
2. Fallback heuristic: largest file by size (if available), else shortest title
3. Final fallback: first PDF attachment in the list
Args:
attachments: List of attachment dicts from load_export_rows().
Returns:
Tuple of (main_pdf_attachment, supplementary_attachments).
main_pdf_attachment may be None if no PDFs found.
supplementary_attachments is a list of all other PDF attachments.
"""
pdf_attachments = [a for a in attachments if isinstance(a, dict) and a.get("contentType") == "application/pdf"]
if not pdf_attachments:
return (None, [])
# Priority 1: Title exactly equals "PDF"
for att in pdf_attachments:
if att.get("title") == "PDF":
main = att
supplementary = [a for a in pdf_attachments if a is not main]
return (main, supplementary)
# Priority 2: Largest file by size (if size field is available and differentiated)
sized = [(a, a.get("size", 0) or 0) for a in pdf_attachments]
sized.sort(key=lambda x: x[1], reverse=True)
if sized and sized[0][1] > 0 and (len(sized) == 1 or sized[0][1] > sized[1][1]):
main = sized[0][0]
supplementary = [a for a in pdf_attachments if a is not main]
return (main, supplementary)
# Priority 2b (sizes equal or unavailable): shortest title
titled = [(a, len(str(a.get("title", "")))) for a in pdf_attachments]
titled.sort(key=lambda x: x[1])
main = titled[0][0]
supplementary = [a for a in pdf_attachments if a is not main]
return (main, supplementary)
def load_export_rows(path: Path) -> list[dict]:
data = read_json(path)
if isinstance(data, list):
return data
if isinstance(data, dict) and isinstance(data.get("items"), list):
collection_lookup = build_collection_lookup(data.get("collections", {}))
rows = []
for item in data["items"]:
if item.get("itemType") in {"attachment", "note", "annotation"}:
continue
attachments = []
for attachment in item.get("attachments", []):
if not isinstance(attachment, dict):
continue
raw_path = attachment.get("path", "")
normalized_path, bbt_path_raw, zotero_storage_key = _normalize_attachment_path(raw_path)
# Preserve contentType from BBT if present; fallback to file extension
content_type = attachment.get("contentType", "")
if not content_type and str(normalized_path).lower().endswith(".pdf"):
content_type = "application/pdf"
attachments.append(
{
"path": normalized_path,
"contentType": content_type,
"title": attachment.get("title", ""),
"bbt_path_raw": bbt_path_raw,
"zotero_storage_key": zotero_storage_key,
"size": attachment.get("size", 0) or 0,
}
)
main_pdf, supplementary_pdfs = _identify_main_pdf(attachments)
pdf_path = main_pdf["path"] if main_pdf else ""
bbt_path_raw = main_pdf["bbt_path_raw"] if main_pdf else ""
zotero_storage_key = main_pdf["zotero_storage_key"] if main_pdf else ""
path_error = "not_found" if not main_pdf else ""
supplementary = [a["path"] for a in supplementary_pdfs] if supplementary_pdfs else []
attachment_count = len(attachments)
rows.append(
{
"key": item.get("key") or item.get("itemKey", ""),
"title": item.get("title", ""),
"authors": extract_authors(item),
"creators": item.get("creators", []),
"abstract": item.get("abstractNote", ""),
"journal": item.get("publicationTitle", ""),
"extra": item.get("extra", ""),
"year": _extract_year(item.get("date", "")),
"date": item.get("date", ""),
"doi": item.get("DOI", ""),
"pmid": item.get("PMID", ""),
"collections": resolve_item_collection_paths(item, collection_lookup),
"attachments": attachments,
"pdf_path": pdf_path,
"supplementary": supplementary,
"attachment_count": attachment_count,
"bbt_path_raw": bbt_path_raw,
"zotero_storage_key": zotero_storage_key,
"path_error": path_error,
}
)
return rows
raise ValueError(f"Unsupported export format: {path}")
def compute_final_collection(row: dict) -> str:
user_raw = str(row.get("user_collection", "") or "").strip()
user_resolved = str(row.get("user_collection_resolved", "") or "").strip()
recommended = str(row.get("recommended_collection", "") or "").strip()
if user_raw:
return user_resolved
return recommended
def canonicalize_decision(value: str) -> str:
text = str(value or "").strip()
if text in {"", "待查"}:
return "待定"
if text in {"排除", "不纳入"}:
return "不纳入"
if text == "纳入":
return "纳入"
return "待定"
def candidate_markdown(row: dict) -> str:
row = dict(row)
row["final_collection"] = compute_final_collection(row)
row["decision"] = canonicalize_decision(row.get("decision", ""))
lines = ["---"]
ordered_keys = [
"candidate_id",
"domain",
"title",
"authors",
"year",
"journal",
"doi",
"pmid",
"source",
"requester_skill",
"request_context",
"abstract_short",
"decision",
"recommended_collection",
"recommend_confidence",
"recommend_reason",
"user_collection",
"user_collection_resolved",
"final_collection",
"collection_resolution",
"duplicate_hint",
"existing_zotero_key",
"existing_collections",
"import_status",
"note",
"candidate_source_type",
"source_zotero_key",
"cited_ref_number",
"trigger_sentence",
"source_context",
"task_relevance_reason",
"harvest_priority",
"raw_reference",
"status",
]
row.setdefault("status", "candidate")
for key in ordered_keys:
value = row.get(key, "")
if isinstance(value, list):
lines.append(f"{key}:")
for item in value:
lines.append(f" - {yaml_quote(item)}")
elif value == "":
lines.append(f"{key}:")
elif "\n" in str(value):
lines.extend(
yaml_block(str(value)).copy()
if key == "abstract"
else [f"{key}: |-"] + [f" {line}" for line in str(value).splitlines()]
)
else:
lines.append(f"{key}: {yaml_quote(value)}")
lines.extend(
[
"---",
"",
f"# {row['candidate_id']}",
"",
"候选文献轻量记录,仅用于 Base 决策和 write-back 触发,不是正式文献卡片。",
"",
]
)
return "\n".join(lines)
def generate_review(candidates: list[dict]) -> str:
normalized = []
for row in candidates:
copy = dict(row)
copy["decision"] = canonicalize_decision(copy.get("decision", ""))
normalized.append(copy)
include = [c for c in normalized if c.get("decision") == "纳入"]
exclude = [c for c in normalized if c.get("decision") == "不纳入"]
lines = [
"# 本轮候选总览",
"",
"## 检索背景",
"",
f"- 候选数量:{len(normalized)}",
f"- 建议纳入:{len(include)}",
f"- 不纳入:{len(exclude)}",
"",
"## 总体判断",
"",
"- 当前候选池已经按决策状态分层,可直接进入 Base 处理。",
"",
"## 推荐优先纳入",
"",
]
if include:
for row in include:
lines.extend(
[
f"### {row['candidate_id']}",
"",
f"- 标题:{row['title']}",
f"- 推荐分类:`{compute_final_collection(row)}`",
f"- 理由:{row.get('recommend_reason', '')}",
"",
]
)
else:
lines.extend(["- 暂无", ""])
lines.extend(["## 不纳入", ""])
if exclude:
for row in exclude:
lines.extend(
[
f"### {row['candidate_id']}",
"",
f"- 标题:{row['title']}",
f"- 理由:{row.get('recommend_reason', '')}",
"",
]
)
else:
lines.extend(["- 暂无", ""])
lines.extend(["## 下一步", "", "1. 在 Base 中确认决策。", "2. 对纳入项执行 write-back。", "3. 刷新正式索引。", ""])
return "\n".join(lines)
DEEP_READING_HEADER = "## 🔍 精读"
def extract_preserved_deep_reading(text: str) -> str:
"""Extract the `## 🔍 精读` section by matching it as a real markdown header.
Uses regex to ensure we match `## 🔍 精读` at the start of a line,
avoiding false positives from prose text that merely mentions the string.
"""
if not text:
return ""
match = re.search("^## 🔍 精读\\s*$", text, re.MULTILINE)
if not match:
return ""
start = match.start()
preserved = text[start:].strip()
return preserved
def has_deep_reading_content(text: str) -> bool:
"""Return True only if the deep-reading section passes a three-anchor check.
For deep_reading_status to be "done", ALL three of these must have
substantive content (not just template scaffolding):
1. **Clarity**清晰度 text after the colon
2. **Figure 导读** at least one list item with content
3. **遗留问题** at least one non-empty entry
"""
preserved = extract_preserved_deep_reading(text)
if not preserved:
return False
body = preserved.replace(DEEP_READING_HEADER, "").strip()
if not body:
return False
# Anchor 1: Clarity
clarity_ok = bool(re.search(
r'-\s*\*\*Clarity\*\*(清晰度):(.+)', body
)) and not re.search(
r'-\s*\*\*Clarity\*\*(清晰度):\s*$', body, re.MULTILINE
)
# Anchor 2: Figure 导读 — at least one list item with content after colon
figure_sec = _extract_section(body, r'\*\*Figure 导读\*\*')
figure_ok = False
if figure_sec:
for line in figure_sec.splitlines():
s = line.strip()
if s.startswith('- ') and '' in s:
_, after = s.split('', 1)
if after.strip() and after.strip() != '(待补充)':
figure_ok = True
break
# Anchor 3: 遗留问题 — at least one non-empty line beneath marker
issue_sec = _extract_section(body, r'\*\*遗留问题\*\*')
if not issue_sec:
issue_sec = _extract_section(body, r'####\s*遗留问题')
issue_ok = False
if issue_sec:
dirty = [l.strip() for l in issue_sec.splitlines() if l.strip()]
substantive = [l for l in dirty if l not in ('-', '(待补充)', '', '**遗留问题**')]
issue_ok = bool(substantive)
return clarity_ok and figure_ok and issue_ok
def _extract_section(body: str, section_header: str) -> str | None:
"""Return block of text from *section_header* to the next section break.
Section break is either a ``##`` / ``###`` header or end of string.
"""
m = re.search(
section_header + r'\n*(.*?)(?=\n(?:#{1,3})\s|\Z)',
body, re.DOTALL,
)
if m:
return m.group(1).strip()
return None
def _add_missing_frontmatter_fields(existing_content: str, new_fields: dict[str, str]) -> str:
"""Surgically append missing fields to existing frontmatter without overwriting anything."""
if not existing_content.startswith("---"):
return existing_content
parts = existing_content.split("---", 2)
if len(parts) < 3:
return existing_content
frontmatter = parts[1]
body = parts[2]
lines_to_add = []
for key, value in new_fields.items():
pattern = "^" + re.escape(key) + "\\s*:"
if not re.search(pattern, frontmatter, re.MULTILINE):
lines_to_add.append(f"{key}: {yaml_quote(value)}")
if not lines_to_add:
return existing_content
new_frontmatter = frontmatter.rstrip("\n") + "\n" + "\n".join(lines_to_add) + "\n"
return f"---{new_frontmatter}---{body}"
def update_frontmatter_field(content: str, key: str, value: str) -> str:
"""Update an existing frontmatter field value, or add if missing."""
if not content.startswith("---"):
return content
pattern = "^" + re.escape(key) + "\\s*:.*$"
replacement = f"{key}: {yaml_quote(value)}"
new_content, count = re.subn(pattern, replacement, content, flags=re.MULTILINE, count=1)
if count == 0:
new_content = _add_missing_frontmatter_fields(content, {key: value})
return new_content
def load_control_actions(paths: dict[str, Path]) -> dict[str, dict]:
actions = {}
@ -736,7 +208,7 @@ def load_control_actions(paths: dict[str, Path]) -> dict[str, dict]:
return actions
def run_selection_sync(vault: Path, verbose: bool = False) -> int:
def run_selection_sync(vault: Path, verbose: bool = False, json_output: bool = False) -> int | dict:
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.ocr import validate_ocr_meta
@ -795,6 +267,8 @@ def run_selection_sync(vault: Path, verbose: bool = False) -> int:
# Formal notes now carry workflow flags (do_ocr, analyze) directly.
# Existing library-records are migrated via Phase 40 logic.
updated += 1
if json_output:
return {"new": written, "updated": updated, "skipped": 0, "failed": 0, "errors": []}
print(f"selection-sync: wrote {written} records, updated {updated} records")
return 0
@ -1720,7 +1194,7 @@ def migrate_to_workspace(vault: Path, paths: dict) -> int:
return migrated
def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool = False) -> int:
def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool = False, json_output: bool = False) -> int:
"""Refresh the canonical asset index.
Default behavior: full rebuild. This is the safe default because
@ -1732,6 +1206,7 @@ def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool =
vault: Path to the vault root.
verbose: If True, print detailed progress.
rebuild_index: If True, force full rebuild (default: True for sync).
json_output: If True, suppress human-readable print output.
"""
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.ocr import validate_ocr_meta
@ -1800,7 +1275,7 @@ def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool =
deleted_count += 1
except Exception:
pass
if deleted_count > 0:
if deleted_count > 0 and not json_output:
print(f"index-refresh: cleaned {deleted_count} orphaned records in {domain}")
# Clean up flat notes: delete only if confirmed by canonical index + workspace
@ -1831,10 +1306,10 @@ def run_index_refresh(vault: Path, verbose: bool = False, rebuild_index: bool =
cleaned_flat += 1
except Exception:
pass
if cleaned_flat > 0:
if cleaned_flat > 0 and not json_output:
print(f"index-refresh: cleaned {cleaned_flat} flat note(s) (migrated to workspace)")
if control_records_dir.exists():
if control_records_dir.exists() and not json_output:
total = sum(1 for _ in control_records_dir.rglob("*.md"))
print(f"index-refresh: {total} formal note(s) in literature")

View file

@ -70,13 +70,14 @@ paperforge = [
[tool.pytest.ini_options]
addopts = "--ignore=tests/sandbox/00_TestVault/ --strict-markers"
testpaths = ["tests/unit", "tests/cli", "tests/e2e", "tests/journey", "tests/chaos"]
testpaths = ["tests/unit", "tests/cli", "tests/e2e", "tests/journey", "tests/chaos", "tests/audit"]
markers = [
"unit: Unit tests (Level 1) — fast, isolated",
"cli: CLI contract tests (Level 2) — subprocess boundary",
"e2e: End-to-end tests (Level 4) — temp vault",
"journey: User journey tests (Level 5) — full workflows",
"chaos: Destructive tests (Level 6) — abnormal scenarios",
"audit: Consistency audit tests — validate L1 mocks against L4 real pipeline output",
"slow: Tests that take >30s (skip during development)",
"snapshot: Tests that use snapshot comparison",
]

View file

@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""Version consistency gate script.
Checks that all version declarations across the codebase match
the canonical version in paperforge.__version__.
Declaration locations checked:
1. paperforge/__init__.py (source of truth)
2. paperforge/plugin/manifest.json
3. paperforge/plugin/versions.json
4. pyproject.toml (dynamic version config)
5. paperforge/plugin/main.js (version fallback strings)
Exit code: 0 if all match, 1 if any mismatch.
"""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
def get_canonical_version() -> str:
"""Read version from paperforge.__init__."""
init_path = REPO_ROOT / "paperforge" / "__init__.py"
content = init_path.read_text(encoding="utf-8")
m = re.search(r'__version__\s*=\s*"([^"]+)"', content)
if not m:
print("[FAIL] Cannot parse __version__ from paperforge/__init__.py")
sys.exit(1)
return m.group(1)
def check_init(canonical: str) -> list[str]:
errors = []
init_path = REPO_ROOT / "paperforge" / "__init__.py"
content = init_path.read_text(encoding="utf-8")
m = re.search(r'__version__\s*=\s*"([^"]+)"', content)
if m and m.group(1) != canonical:
errors.append(f' paperforge/__init__.py: __version__ = "{m.group(1)}" (expected "{canonical}")')
return errors
def check_plugin_manifest(canonical: str) -> list[str]:
errors = []
path = REPO_ROOT / "paperforge" / "plugin" / "manifest.json"
try:
data = json.loads(path.read_text(encoding="utf-8"))
ver = data.get("version")
if ver != canonical:
errors.append(f' paperforge/plugin/manifest.json: version = "{ver}" (expected "{canonical}")')
except (json.JSONDecodeError, FileNotFoundError) as e:
errors.append(f" paperforge/plugin/manifest.json: failed to read — {e}")
return errors
def check_root_manifest(canonical: str) -> list[str]:
errors = []
path = REPO_ROOT / "manifest.json"
try:
data = json.loads(path.read_text(encoding="utf-8"))
ver = data.get("version")
if ver != canonical:
errors.append(f' manifest.json: version = "{ver}" (expected "{canonical}")')
except (json.JSONDecodeError, FileNotFoundError) as e:
errors.append(f" manifest.json: failed to read — {e}")
return errors
def check_versions_json(canonical: str) -> list[str]:
errors = []
path = REPO_ROOT / "paperforge" / "plugin" / "versions.json"
try:
data = json.loads(path.read_text(encoding="utf-8"))
if canonical not in data:
errors.append(f' paperforge/plugin/versions.json: missing entry for version "{canonical}"')
except (json.JSONDecodeError, FileNotFoundError) as e:
errors.append(f" paperforge/plugin/versions.json: failed to read — {e}")
return errors
def check_pyproject_toml() -> list[str]:
errors = []
path = REPO_ROOT / "pyproject.toml"
if not path.exists():
errors.append(" pyproject.toml: not found")
return errors
content = path.read_text(encoding="utf-8")
if 'dynamic = ["version"]' not in content:
errors.append(' pyproject.toml: missing dynamic = ["version"]')
if 'version = {attr = "paperforge.__version__"}' not in content:
errors.append(" pyproject.toml: missing version attr reference")
return errors
def check_main_js(canonical: str) -> list[str]:
errors = []
path = REPO_ROOT / "paperforge" / "plugin" / "main.js"
if not path.exists():
errors.append(" paperforge/plugin/main.js: not found")
return errors
content = path.read_text(encoding="utf-8")
# Find hardcoded version strings in fallback patterns
fallback_pattern = re.compile(r"['\"](\d+\.\d+\.\d+\w*)['\"]")
for i, line in enumerate(content.splitlines(), 1):
stripped = line.strip()
# Skip comments and non-fallback lines
if "manifest.version" not in stripped and "this.manifest" not in stripped:
continue
for m in fallback_pattern.finditer(stripped):
found = m.group(1)
if found == canonical:
continue
errors.append(
f' paperforge/plugin/main.js:{i}: hardcoded version "{found}"'
f' (expected "{canonical}" or dynamic from manifest)'
)
return errors
def main() -> int:
canonical = get_canonical_version()
print(f"Canonical version: {canonical}")
print()
checks: list[tuple[str, list[str]]] = [
("paperforge/__init__.py", check_init(canonical)),
("paperforge/plugin/manifest.json", check_plugin_manifest(canonical)),
("manifest.json", check_root_manifest(canonical)),
("paperforge/plugin/versions.json", check_versions_json(canonical)),
("pyproject.toml (dynamic version config)", check_pyproject_toml()),
("paperforge/plugin/main.js (fallback strings)", check_main_js(canonical)),
]
total_errors = 0
for label, errors in checks:
if not errors:
print(f"[PASS] {label}")
else:
print(f"[FAIL] {label}")
for e in errors:
print(e)
total_errors += len(errors)
print()
if total_errors == 0:
print("All version declarations are consistent.")
return 0
else:
print(f"Found {total_errors} version mismatch(es).")
return 1
if __name__ == "__main__":
sys.exit(main())

View file

@ -1,5 +0,0 @@
{
"vault": "<VAULT>",
"worker_script": "D:\\L\\Med\\Research\\99_System\\LiteraturePipeline\\github-release\\paperforge\\worker\\__init__.py",
"ld_deep_script": "D:\\L\\Med\\Research\\99_System\\LiteraturePipeline\\github-release\\paperforge\\skills\\literature-qa\\scripts\\ld_deep.py"
}

View file

@ -1,23 +0,0 @@
{
"version": "<VERSION>",
"vault": "<VAULT>",
"system_dir": "System",
"resources_dir": "Resources",
"exports": 0,
"domains": 0,
"total_papers": 0,
"formal_notes": 0,
"bases": 1,
"ocr": {
"total": 0,
"pending": 0,
"processing": 0,
"done": 0,
"failed": 0
},
"path_errors": 0,
"env_configured": true,
"lifecycle_level_counts": {},
"health_aggregate": {},
"maturity_distribution": {}
}

View file

@ -25,16 +25,12 @@ def normalize_snapshot(output: str, vault_path: str | None = None) -> str:
"""
result = output
# Normalize vault paths (absolute paths containing /tmp/ or \\tmp\\)
# Normalize vault paths — use literal replace (not regex) to handle JSON-escaped paths
if vault_path:
escaped = re.escape(str(vault_path))
result = re.sub(escaped, "<VAULT>", result)
# Fallback: any path containing /tmp/pf_vault_ (Unix) or pf_vault_ (Windows)
result = result.replace(str(vault_path), "<VAULT>")
# Fallback: any path matching common temp vault patterns
result = re.sub(r'[A-Za-z]:[\\/][^\s"\']*pf_vault_[a-z0-9_]+', "<VAULT>", result)
result = re.sub(r'/tmp/pf_vault_[^/\s"\']+', "<VAULT>", result)
result = re.sub(r'\\\\temp\\\\pf_vault_[^\\\\\s"\']+', "<VAULT>", result)
# Broader: any string ending in pf_vault_XXXXX as a path component
result = re.sub(r'[A-Za-z]:[\\/][^\s"\']*pf_vault_[a-z0-9]+', "<VAULT>", result)
# Normalize ISO timestamps: 2026-05-08T12:34:56.789012+00:00
result = re.sub(

View file

@ -14,29 +14,21 @@ from .test_contract_helpers import (
class TestPathsJson:
"""Contract: 'paperforge paths --json' returns stable JSON."""
"""Contract: 'paperforge paths --json' returns valid JSON with correct value types."""
REQUIRED_KEYS = {"vault", "worker_script", "ld_deep_script"}
def test_paths_json_valid(self, cli_invoker):
"""paths --json outputs valid JSON with expected keys."""
def test_paths_json_value_semantics(self, cli_invoker):
"""paths --json returns paths with correct value types (all non-empty strings)."""
result = cli_invoker(["paths", "--json"])
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr}"
data = assert_valid_json(result.stdout)
assert_json_shape(data, self.REQUIRED_KEYS)
def test_paths_json_snapshot(self, cli_invoker, snapshot):
"""paths --json matches snapshot (normalized)."""
result = cli_invoker(["paths", "--json"])
assert result.returncode == 0
# Parse and re-serialize for consistent formatting
data = json.loads(result.stdout)
vault = data.get("vault", "")
normalized = normalize_snapshot(
json.dumps(data, indent=2, ensure_ascii=False), vault
)
snapshot.assert_match(normalized, "paths_json/default_config.json")
for key in self.REQUIRED_KEYS:
val = data[key]
assert isinstance(val, str), f"{key} should be string, got {type(val).__name__}"
assert len(val) > 0, f"{key} should not be empty"
assert data.get("vault", "").startswith("<") or len(data.get("vault", "")) > 0
def test_paths_no_json_text_output(self, cli_invoker):
"""paths without --json outputs human-readable text."""
@ -48,7 +40,7 @@ class TestPathsJson:
class TestStatusJson:
"""Contract: 'paperforge status --json' returns stable JSON."""
"""Contract: 'paperforge status --json' returns JSON with correct value semantics."""
REQUIRED_KEYS = {"total_papers", "version", "vault", "system_dir", "resources_dir"}
OPTIONAL_KEYS = {
@ -57,23 +49,30 @@ class TestStatusJson:
"lifecycle_level_counts", "health_aggregate", "maturity_distribution",
}
def test_status_json_on_empty_vault(self, cli_invoker):
"""status --json on empty vault returns valid JSON with all contract keys."""
def test_status_json_value_semantics(self, cli_invoker):
"""status --json returns valid JSON with all contract keys and correct value types."""
result = cli_invoker(["status", "--json"])
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr}"
data = assert_valid_json(result.stdout)
assert_json_shape(data, self.REQUIRED_KEYS, self.OPTIONAL_KEYS)
# Empty vault should have 0 papers
assert data.get("total_papers", -1) >= 0
total = data.get("total_papers", -1)
assert isinstance(total, int) and total >= 0, f"total_papers should be non-negative int, got {total}"
ver = data.get("version", "")
assert isinstance(ver, str) and len(ver) > 0, f"version should be non-empty string, got {ver}"
assert data.get("system_dir", "") == "System", f"system_dir should be 'System', got {data.get('system_dir')}"
assert data.get("resources_dir", "") == "Resources", f"resources_dir should be 'Resources', got {data.get('resources_dir')}"
if "ocr" in data:
ocr = data["ocr"]
for field in ("total", "pending", "processing", "done", "failed"):
assert isinstance(ocr.get(field), int), f"ocr.{field} should be int, got {type(ocr.get(field)).__name__}"
def test_status_json_snapshot(self, cli_invoker, snapshot):
"""status --json matches snapshot (normalized)."""
def test_status_json_snapshot_regression(self, cli_invoker, snapshot):
"""status --json snapshot for regression detection (value semantics tested separately)."""
result = cli_invoker(["status", "--json"])
assert result.returncode == 0
try:
data = json.loads(result.stdout)
except json.JSONDecodeError:
# status --json may not output JSON in all vault states
pytest.skip("status --json did not produce valid JSON")
vault = data.get("vault", "")
normalized = normalize_snapshot(

0
tests/unit/__init__.py Normal file
View file

View file

View file

@ -0,0 +1,391 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from paperforge.adapters.bbt import (
_identify_main_pdf,
_normalize_attachment_path,
collection_fields,
extract_authors,
load_export_rows,
resolve_item_collection_paths,
)
class TestNormalizeAttachmentPath:
def test_storage_prefix_passthrough(self) -> None:
result = _normalize_attachment_path("storage:ABCD1234/file.pdf")
assert result == ("storage:ABCD1234/file.pdf", "storage:ABCD1234/file.pdf", "ABCD1234")
def test_storage_prefix_backslash_normalization(self) -> None:
result = _normalize_attachment_path("storage:ABCD1234\\file.pdf")
assert result == ("storage:ABCD1234/file.pdf", "storage:ABCD1234\\file.pdf", "ABCD1234")
def test_absolute_windows_path_with_zotero_storage(self) -> None:
result = _normalize_attachment_path(r"D:\Zotero\storage\ABCD1234\paper.pdf")
assert result == ("storage:ABCD1234/paper.pdf", r"D:\Zotero\storage\ABCD1234\paper.pdf", "ABCD1234")
def test_absolute_windows_path_not_in_zotero(self) -> None:
result = _normalize_attachment_path(r"D:\other\paper.pdf")
assert result == ("absolute:D:\\other\\paper.pdf", r"D:\other\paper.pdf", "")
def test_bare_relative_path(self) -> None:
result = _normalize_attachment_path("EFGH5678/doc.pdf")
assert result == ("storage:EFGH5678/doc.pdf", "EFGH5678/doc.pdf", "EFGH5678")
def test_chinese_filename(self) -> None:
result = _normalize_attachment_path(r"D:\Zotero\storage\ABCD1234\中文论文.pdf")
assert result == ("storage:ABCD1234/中文论文.pdf", r"D:\Zotero\storage\ABCD1234\中文论文.pdf", "ABCD1234")
def test_path_with_spaces(self) -> None:
result = _normalize_attachment_path(r"D:\Zotero\storage\ABCD1234\my paper.pdf")
assert result == ("storage:ABCD1234/my paper.pdf", r"D:\Zotero\storage\ABCD1234\my paper.pdf", "ABCD1234")
def test_empty_path_returns_empty(self) -> None:
assert _normalize_attachment_path("") == ("", "", "")
assert _normalize_attachment_path(None) == ("", "", "")
assert _normalize_attachment_path(" ") == ("", "", "")
class TestIdentifyMainPdf:
def test_single_pdf_becomes_main(self) -> None:
atts = [{"contentType": "application/pdf", "title": "Full Text", "path": "storage:K/key.pdf"}]
main, supp = _identify_main_pdf(atts)
assert main is atts[0]
assert supp == []
def test_title_pdf_is_selected(self) -> None:
atts = [
{"contentType": "application/pdf", "title": "PDF", "path": "storage:K/main.pdf"},
{"contentType": "application/pdf", "title": "Supplement", "path": "storage:K/supp.pdf"},
]
main, supp = _identify_main_pdf(atts)
assert main["title"] == "PDF"
assert len(supp) == 1
assert supp[0]["title"] == "Supplement"
def test_largest_file_by_size(self) -> None:
atts = [
{"contentType": "application/pdf", "title": "Small", "size": 100},
{"contentType": "application/pdf", "title": "Large", "size": 999999},
]
main, supp = _identify_main_pdf(atts)
assert main["title"] == "Large"
assert len(supp) == 1
def test_shortest_title_when_sizes_equal(self) -> None:
atts = [
{"contentType": "application/pdf", "title": "A Longer Title Here", "size": 500},
{"contentType": "application/pdf", "title": "Short", "size": 500},
]
main, supp = _identify_main_pdf(atts)
assert main["title"] == "Short"
def test_no_attachments_returns_none(self) -> None:
main, supp = _identify_main_pdf([])
assert main is None
assert supp == []
def test_no_pdf_attachments(self) -> None:
atts = [{"contentType": "text/html", "title": "Snapshot"}]
main, supp = _identify_main_pdf(atts)
assert main is None
assert supp == []
def test_no_size_field_falls_back_to_shortest_title(self) -> None:
atts = [
{"contentType": "application/pdf", "title": "Long Title Document"},
{"contentType": "application/pdf", "title": "AB"},
]
main, supp = _identify_main_pdf(atts)
assert main["title"] == "AB"
def test_first_pdf_fallback(self) -> None:
atts = [
{"contentType": "application/pdf", "title": "First"},
{"contentType": "application/pdf", "title": "Second"},
]
main, supp = _identify_main_pdf(atts)
assert main["title"] == "First"
def test_non_dict_item_skipped(self) -> None:
atts = [
{"contentType": "application/pdf", "title": "PDF", "path": "storage:K/main.pdf"},
"not a dict",
]
main, supp = _identify_main_pdf(atts)
assert main is not None
assert main["title"] == "PDF"
class TestExtractAuthors:
def test_single_author(self) -> None:
item = {"creators": [{"creatorType": "author", "firstName": "John", "lastName": "Doe"}]}
assert extract_authors(item) == ["John Doe"]
def test_multiple_authors(self) -> None:
item = {
"creators": [
{"creatorType": "author", "firstName": "Alice", "lastName": "Smith"},
{"creatorType": "author", "firstName": "Bob", "lastName": "Jones"},
]
}
assert extract_authors(item) == ["Alice Smith", "Bob Jones"]
def test_skips_non_author_creators(self) -> None:
item = {
"creators": [
{"creatorType": "author", "firstName": "Only", "lastName": "Author"},
{"creatorType": "editor", "firstName": "Not", "lastName": "Included"},
]
}
assert extract_authors(item) == ["Only Author"]
def test_empty_creators(self) -> None:
assert extract_authors({"creators": []}) == []
def test_missing_creators_key(self) -> None:
assert extract_authors({}) == []
def test_author_with_no_first_name(self) -> None:
item = {"creators": [{"creatorType": "author", "lastName": "Doe"}]}
assert extract_authors(item) == ["Doe"]
def test_author_with_no_last_name(self) -> None:
item = {"creators": [{"creatorType": "author", "firstName": "John"}]}
assert extract_authors(item) == ["John"]
def test_author_with_name_field(self) -> None:
item = {"creators": [{"creatorType": "author", "name": "Institutional Author"}]}
assert extract_authors(item) == ["Institutional Author"]
class TestCollectionFields:
def test_single_path(self) -> None:
result = collection_fields(["骨科/脊柱"])
assert result["collections"] == ["骨科/脊柱"]
assert "骨科" in result["collection_tags"]
assert "脊柱" in result["collection_tags"]
assert result["collection_group"] == ["骨科/脊柱"]
def test_multiple_paths(self) -> None:
paths = ["骨科/脊柱/颈椎", "骨科/脊柱"]
result = collection_fields(paths)
assert len(result["collections"]) == 2
assert "骨科" in result["collection_tags"]
assert "脊柱" in result["collection_tags"]
assert "颈椎" in result["collection_tags"]
def test_empty_list(self) -> None:
result = collection_fields([])
assert result["collections"] == []
assert result["collection_tags"] == []
assert result["collection_group"] == []
def test_nested_path(self) -> None:
result = collection_fields(["骨科/脊柱/颈椎/后路"])
assert result["collections"] == ["骨科/脊柱/颈椎/后路"]
assert result["collection_tags"] == ["骨科", "脊柱", "颈椎", "后路"]
def test_duplicate_tags_deduplicated(self) -> None:
result = collection_fields(["骨科/脊柱", "骨科/创伤"])
assert result["collection_tags"] == ["骨科", "脊柱", "创伤"]
def test_primary_is_deepest_path(self) -> None:
result = collection_fields(["运动医学/膝关节", "运动医学"])
assert result["collection_group"] == ["运动医学/膝关节"]
def test_whitespace_in_paths(self) -> None:
result = collection_fields([" 骨科 / 脊柱 "])
assert "骨科" in result["collection_tags"]
assert "脊柱" in result["collection_tags"]
class TestResolveItemCollectionPaths:
def test_single_collection(self) -> None:
item = {"collections": ["col1"]}
lookup = {"path_by_key": {"col1": "骨科/脊柱"}, "paths_by_item_id": {}}
assert resolve_item_collection_paths(item, lookup) == ["骨科/脊柱"]
def test_multiple_collections(self) -> None:
item = {"collections": ["col1", "col2"]}
lookup = {
"path_by_key": {"col1": "骨科/脊柱", "col2": "运动医学/膝关节"},
"paths_by_item_id": {},
}
result = resolve_item_collection_paths(item, lookup)
assert "骨科/脊柱" in result
assert "运动医学/膝关节" in result
def test_item_id_paths(self) -> None:
item = {"itemID": 42}
lookup = {"path_by_key": {}, "paths_by_item_id": {42: ["骨科/创伤"]}}
assert resolve_item_collection_paths(item, lookup) == ["骨科/创伤"]
def test_combined_collections_and_item_id(self) -> None:
item = {"collections": ["col1"], "itemID": 42}
lookup = {
"path_by_key": {"col1": "骨科/脊柱"},
"paths_by_item_id": {42: ["骨科/脊柱/颈椎"]},
}
result = resolve_item_collection_paths(item, lookup)
assert "骨科/脊柱" in result
assert "骨科/脊柱/颈椎" in result
def test_empty_item(self) -> None:
item = {}
lookup = {"path_by_key": {}, "paths_by_item_id": {}}
assert resolve_item_collection_paths(item, lookup) == []
def test_unknown_key_falls_back_to_key(self) -> None:
item = {"collections": ["unknown_key"]}
lookup = {"path_by_key": {}, "paths_by_item_id": {}}
assert resolve_item_collection_paths(item, lookup) == ["unknown_key"]
def test_deduplication(self) -> None:
item = {"collections": ["col1"], "itemID": 1}
lookup = {
"path_by_key": {"col1": "骨科/脊柱"},
"paths_by_item_id": {1: ["骨科/脊柱"]},
}
assert resolve_item_collection_paths(item, lookup) == ["骨科/脊柱"]
def test_no_collections_key(self) -> None:
item = {"key": "ABC123"}
lookup = {"path_by_key": {}, "paths_by_item_id": {}}
assert resolve_item_collection_paths(item, lookup) == []
class TestLoadExportRows:
def test_valid_json_with_items(self, tmp_path: Path) -> None:
data = {
"items": [
{
"key": "ABC123",
"itemType": "journalArticle",
"title": "Test Article",
"creators": [{"creatorType": "author", "firstName": "John", "lastName": "Doe"}],
"date": "2023",
"DOI": "10.1234/test",
"PMID": "12345678",
"publicationTitle": "Test Journal",
"abstractNote": "An abstract.",
"collections": ["col1"],
"attachments": [
{
"path": "storage:ABCD1234/paper.pdf",
"contentType": "application/pdf",
"title": "PDF",
}
],
}
],
"collections": {"col1": {"name": "骨科", "parent": "", "items": []}},
}
path = tmp_path / "export.json"
path.write_text(json.dumps(data), encoding="utf-8")
rows = load_export_rows(path)
assert len(rows) == 1
assert rows[0]["key"] == "ABC123"
assert rows[0]["title"] == "Test Article"
assert rows[0]["authors"] == ["John Doe"]
assert rows[0]["year"] == "2023"
assert rows[0]["doi"] == "10.1234/test"
assert rows[0]["pmid"] == "12345678"
assert rows[0]["pdf_path"] == "storage:ABCD1234/paper.pdf"
assert rows[0]["collections"] == ["骨科"]
def test_list_format_data(self, tmp_path: Path) -> None:
data = [
{"key": "ITEM1", "title": "Item 1"},
{"key": "ITEM2", "title": "Item 2"},
]
path = tmp_path / "list.json"
path.write_text(json.dumps(data), encoding="utf-8")
rows = load_export_rows(path)
assert len(rows) == 2
assert rows[0]["key"] == "ITEM1"
assert rows[1]["key"] == "ITEM2"
def test_empty_json_list(self, tmp_path: Path) -> None:
path = tmp_path / "empty.json"
path.write_text("[]", encoding="utf-8")
assert load_export_rows(path) == []
def test_file_not_found(self, tmp_path: Path) -> None:
path = tmp_path / "nonexistent.json"
with pytest.raises(FileNotFoundError):
load_export_rows(path)
def test_skips_attachment_notes_and_annotations(self, tmp_path: Path) -> None:
data = {
"items": [
{"key": "ART1", "itemType": "journalArticle", "title": "Real Article", "creators": [], "date": "", "collections": [], "attachments": []},
{"key": "ATT1", "itemType": "attachment", "title": "An Attachment"},
{"key": "NOT1", "itemType": "note", "title": "A Note"},
{"key": "ANN1", "itemType": "annotation", "title": "An Annotation"},
],
"collections": {},
}
path = tmp_path / "mixed.json"
path.write_text(json.dumps(data), encoding="utf-8")
rows = load_export_rows(path)
assert len(rows) == 1
assert rows[0]["key"] == "ART1"
def test_unsupported_format_raises_error(self, tmp_path: Path) -> None:
path = tmp_path / "bad.json"
path.write_text('"just a string"', encoding="utf-8")
with pytest.raises(ValueError, match="Unsupported export format"):
load_export_rows(path)
def test_pdf_content_type_fallback(self, tmp_path: Path) -> None:
data = {
"items": [
{
"key": "ABC123",
"itemType": "journalArticle",
"title": "Test",
"date": "",
"creators": [],
"collections": [],
"attachments": [
{
"path": "storage:KEY/file.pdf",
}
],
}
],
"collections": {},
}
path = tmp_path / "ctype.json"
path.write_text(json.dumps(data), encoding="utf-8")
rows = load_export_rows(path)
assert rows[0]["attachments"][0]["contentType"] == "application/pdf"
def test_supplementary_pdfs_separated(self, tmp_path: Path) -> None:
data = {
"items": [
{
"key": "ABC123",
"itemType": "journalArticle",
"title": "Test",
"date": "",
"creators": [],
"collections": [],
"attachments": [
{"path": "storage:K/main.pdf", "contentType": "application/pdf", "title": "PDF"},
{"path": "storage:K/supp1.pdf", "contentType": "application/pdf", "title": "Supplement 1"},
],
}
],
"collections": {},
}
path = tmp_path / "supp.json"
path.write_text(json.dumps(data), encoding="utf-8")
rows = load_export_rows(path)
assert rows[0]["pdf_path"] == "storage:K/main.pdf"
assert rows[0]["supplementary"] == ["storage:K/supp1.pdf"]

View file

@ -0,0 +1,249 @@
from __future__ import annotations
from pathlib import Path
from paperforge.adapters.obsidian_frontmatter import (
_add_missing_frontmatter_fields,
_extract_section,
_legacy_control_flags,
_read_frontmatter_bool_from_text,
_read_frontmatter_optional_bool_from_text,
canonicalize_decision,
candidate_markdown,
compute_final_collection,
extract_preserved_deep_reading,
generate_review,
has_deep_reading_content,
read_frontmatter_dict,
update_frontmatter_field,
)
class TestReadFrontmatterBoolFromText:
def test_true_returns_true(self) -> None:
text = "some_key: true\nother: thing"
assert _read_frontmatter_bool_from_text(text, "some_key") is True
def test_false_returns_false(self) -> None:
text = "some_key: false\nother: thing"
assert _read_frontmatter_bool_from_text(text, "some_key") is False
def test_missing_returns_default(self) -> None:
text = "other: thing"
assert _read_frontmatter_bool_from_text(text, "missing") is False
assert _read_frontmatter_bool_from_text(text, "missing", True) is True
class TestReadFrontmatterOptionalBoolFromText:
def test_true_returns_true(self) -> None:
text = "do_ocr: true"
assert _read_frontmatter_optional_bool_from_text(text, "do_ocr") is True
def test_false_returns_false(self) -> None:
text = "do_ocr: false"
assert _read_frontmatter_optional_bool_from_text(text, "do_ocr") is False
def test_missing_returns_none(self) -> None:
text = "other: thing"
assert _read_frontmatter_optional_bool_from_text(text, "missing") is None
class TestReadFrontmatterDict:
def test_valid_yaml(self) -> None:
text = "---\nkey: value\nnum: 42\n---\nbody"
result = read_frontmatter_dict(text)
assert result == {"key": "value", "num": 42}
def test_malformed_yaml_fallback(self) -> None:
text = "---\nkey: value\n: : malformed\n---\nbody"
result = read_frontmatter_dict(text)
assert "key" not in result or result.get("key") is not None
def test_no_frontmatter_returns_empty(self) -> None:
text = "plain text body\nno frontmatter here"
result = read_frontmatter_dict(text)
assert result == {}
def test_nested_values(self) -> None:
text = "---\nnested:\n inner: val\n num: 42\n---"
result = read_frontmatter_dict(text)
assert result["nested"]["inner"] == "val"
assert result["nested"]["num"] == 42
class TestLegacyControlFlags:
def test_no_library_records_path(self, tmp_path: Path) -> None:
paths = {"library_records": tmp_path / "nonexistent"}
result = _legacy_control_flags(paths, "ABCDEFG")
assert result == {"do_ocr": None, "analyze": None}
def test_no_record_found(self, tmp_path: Path) -> None:
records_dir = tmp_path / "records"
records_dir.mkdir()
paths = {"library_records": records_dir}
result = _legacy_control_flags(paths, "NOKEY")
assert result == {"do_ocr": None, "analyze": None}
def test_record_read_success(self, tmp_path: Path) -> None:
records_dir = tmp_path / "records"
records_dir.mkdir()
record_file = records_dir / "ABCDEFG.md"
record_file.write_text("---\ndo_ocr: true\nanalyze: false\n---", encoding="utf-8")
paths = {"library_records": records_dir}
result = _legacy_control_flags(paths, "ABCDEFG")
assert result == {"do_ocr": True, "analyze": False}
class TestComputeFinalCollection:
def test_user_collection_used_when_present(self) -> None:
row = {"user_collection": "骨科/脊柱", "user_collection_resolved": "骨科/脊柱/颈椎", "recommended_collection": "骨科/创伤"}
assert compute_final_collection(row) == "骨科/脊柱/颈椎"
def test_recommended_fallback(self) -> None:
row = {"user_collection": "", "user_collection_resolved": "", "recommended_collection": "骨科/创伤"}
assert compute_final_collection(row) == "骨科/创伤"
def test_empty_returns_empty(self) -> None:
row = {"user_collection": "", "user_collection_resolved": "", "recommended_collection": ""}
assert compute_final_collection(row) == ""
class TestCanonicalizeDecision:
def test_empty_returns_daidin(self) -> None:
assert canonicalize_decision("") == "待定"
def test_daicha_returns_daidin(self) -> None:
assert canonicalize_decision("待查") == "待定"
def test_paichu_returns_bunaron(self) -> None:
assert canonicalize_decision("排除") == "不纳入"
def test_bunaron_returns_bunaron(self) -> None:
assert canonicalize_decision("不纳入") == "不纳入"
def test_naru_returns_naru(self) -> None:
assert canonicalize_decision("纳入") == "纳入"
class TestCandidateMarkdown:
def test_complete_row(self) -> None:
row = {"candidate_id": "C001", "domain": "骨科", "title": "Test Paper", "authors": ["Author A"], "year": 2024}
result = candidate_markdown(row)
assert result.startswith("---")
assert '"C001"' in result
assert '"Test Paper"' in result
def test_minimum_row(self) -> None:
row = {"candidate_id": "C002", "title": "Minimal"}
result = candidate_markdown(row)
assert result.startswith("---")
assert "C002" in result
def test_missing_optional_fields(self) -> None:
row = {"candidate_id": "C003", "title": "Partial", "doi": "10.1234/test"}
result = candidate_markdown(row)
assert "doi" in result
assert result.startswith("---")
class TestGenerateReview:
def test_include_and_exclude(self) -> None:
candidates = [
{"candidate_id": "C001", "title": "Keep", "decision": "纳入", "recommended_collection": "骨科/脊柱"},
{"candidate_id": "C002", "title": "Drop", "decision": "排除"},
]
result = generate_review(candidates)
assert "Keep" in result
assert "Drop" in result
assert "建议纳入1" in result
assert "不纳入1" in result
def test_empty_list(self) -> None:
result = generate_review([])
assert "候选数量0" in result
class TestExtractPreservedDeepReading:
def test_with_section(self) -> None:
text = "some preamble\n## 🔍 精读\nDeep reading content here\nmore content"
result = extract_preserved_deep_reading(text)
assert "## 🔍 精读" in result
assert "Deep reading content" in result
def test_without_section(self) -> None:
text = "some content\nno deep reading section"
result = extract_preserved_deep_reading(text)
assert result == ""
def test_empty_text(self) -> None:
assert extract_preserved_deep_reading("") == ""
class TestHasDeepReadingContent:
def test_full_content_returns_true(self) -> None:
text = """preamble
## 🔍 精读
- **Clarity**清晰度The paper clearly describes the method
- **Figure 导读**
- Figure 1Shows the main result
- **遗留问题**
- Need more validation"""
assert has_deep_reading_content(text) is True
def test_empty_body_returns_false(self) -> None:
text = "## 🔍 精读"
assert has_deep_reading_content(text) is False
def test_no_deep_reading_section_returns_false(self) -> None:
text = "no section here"
assert has_deep_reading_content(text) is False
class TestExtractSection:
def test_section_extracted(self) -> None:
body = "Some text\n**Figure 导读**\n- Figure 1: content\n## Next section"
result = _extract_section(body, r'\*\*Figure 导读\*\*')
assert result is not None
assert "Figure 1" in result
def test_no_match_returns_none(self) -> None:
body = "no matching header here"
result = _extract_section(body, r'\*\*Missing\*\*')
assert result is None
class TestAddMissingFrontmatterFields:
def test_fields_added(self) -> None:
content = "---\nexisting: val\n---\nbody"
result = _add_missing_frontmatter_fields(content, {"new_key": "new_val"})
assert "new_key:" in result
assert "existing: val" in result
def test_all_present_no_change(self) -> None:
content = "---\nexisting: val\n---\nbody"
result = _add_missing_frontmatter_fields(content, {"existing": "val"})
assert result == content
def test_no_frontmatter_returns_unchanged(self) -> None:
content = "no frontmatter here"
result = _add_missing_frontmatter_fields(content, {"key": "val"})
assert result == content
class TestUpdateFrontmatterField:
def test_overwrite_existing(self) -> None:
content = "---\nkey: old\nother: val\n---\nbody"
result = update_frontmatter_field(content, "key", "new")
assert '"new"' in result
assert "old" not in result
def test_add_new_key(self) -> None:
content = "---\none: val\n---\nbody"
result = update_frontmatter_field(content, "new_key", "new_val")
assert "new_key:" in result
assert "one: val" in result
def test_no_frontmatter_returns_unchanged(self) -> None:
content = "no frontmatter"
result = update_frontmatter_field(content, "key", "val")
assert result == content

View file

@ -0,0 +1,133 @@
from __future__ import annotations
from pathlib import Path
from unittest.mock import patch
from paperforge.adapters.zotero_paths import (
absolutize_vault_path,
obsidian_wikilink_for_path,
obsidian_wikilink_for_pdf,
)
class TestObsidianWikilinkForPdf:
"""Tests for obsidian_wikilink_for_pdf path resolution."""
def test_empty_path_returns_empty(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
assert obsidian_wikilink_for_pdf("", vault) == ""
assert obsidian_wikilink_for_pdf(None, vault) == ""
assert obsidian_wikilink_for_pdf(" ", vault) == ""
def test_storage_prefix_resolves_through_zotero_dir(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
zotero = tmp_path / "zotero_data"
zotero.mkdir()
result = obsidian_wikilink_for_pdf("storage:ABCD1234/paper.pdf", vault, zotero)
assert result == f"[[{(zotero / 'storage' / 'ABCD1234' / 'paper.pdf').as_posix()}]]"
def test_absolute_windows_path_outside_vault(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
abs_path = tmp_path / "outside" / "file.pdf"
abs_path.parent.mkdir(parents=True)
abs_path.write_text("content")
result = obsidian_wikilink_for_pdf(str(abs_path), vault)
assert "outside" in result
assert "file.pdf" in result
assert result.startswith("[[")
assert result.endswith("]]")
def test_cjk_filename_with_spaces(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
zotero = tmp_path / "zotero_data"
zotero.mkdir()
result = obsidian_wikilink_for_pdf("storage:KEY/中文 文件.pdf", vault, zotero)
assert "中文" in result
assert "文件" in result
def test_bare_key_no_prefix_resolves_relative_to_vault(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
result = obsidian_wikilink_for_pdf("ABCD1234/paper.pdf", vault)
assert result == "[[ABCD1234/paper.pdf]]"
def test_storage_prefix_without_zotero_dir_falls_back(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
result = obsidian_wikilink_for_pdf("storage:ABCD/paper.pdf", vault)
assert "ABCD" in result
class TestAbsolutizeVaultPath:
"""Tests for absolutize_vault_path."""
def test_empty_path_returns_empty(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
assert absolutize_vault_path(vault, "") == ""
assert absolutize_vault_path(vault, " ") == ""
def test_relative_path_resolution(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
result = absolutize_vault_path(vault, "subdir/file.pdf")
expected = str((vault / "subdir" / "file.pdf").resolve())
assert result == expected
def test_relative_with_backslashes(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
result = absolutize_vault_path(vault, "subdir\\file.pdf")
expected = str((vault / "subdir" / "file.pdf").resolve())
assert result == expected
def test_absolute_path_passthrough(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
abs_path = str(tmp_path / "outside" / "file.pdf")
result = absolutize_vault_path(vault, abs_path)
assert result == abs_path
def test_junction_resolution_applied_when_flag_set(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
with patch("paperforge.pdf_resolver.resolve_junction") as mock_resolve:
mock_resolve.return_value = tmp_path / "resolved" / "file.pdf"
result = absolutize_vault_path(vault, "sub/file.pdf", resolve_junction=True)
assert "resolved" in result
def test_junction_not_applied_when_flag_false(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
with patch("paperforge.pdf_resolver.resolve_junction") as mock_resolve:
result = absolutize_vault_path(vault, "sub/file.pdf", resolve_junction=False)
mock_resolve.assert_not_called()
class TestObsidianWikilinkForPath:
"""Tests for obsidian_wikilink_for_path."""
def test_empty_path_returns_empty(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
assert obsidian_wikilink_for_path(vault, "") == ""
assert obsidian_wikilink_for_path(vault, " ") == ""
def test_relative_path_to_wikilink(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
result = obsidian_wikilink_for_path(vault, "subdir/paper.pdf")
assert result == "[[subdir/paper.pdf]]"
def test_absolute_path_outside_vault(self, tmp_path: Path) -> None:
vault = tmp_path / "vault"
vault.mkdir()
abs_path = tmp_path / "outside" / "file.pdf"
abs_path.parent.mkdir(parents=True)
abs_path.write_text("content")
result = obsidian_wikilink_for_path(vault, str(abs_path))
assert result == f"[[{abs_path.as_posix()}]]"

View file

View file

@ -0,0 +1,28 @@
"""Unit tests for ErrorCode enum."""
from __future__ import annotations
from paperforge.core.errors import ErrorCode
class TestErrorCodeMembers:
"""ErrorCode enum members match expected string values."""
def test_members_and_values(self) -> None:
assert ErrorCode.PYTHON_NOT_FOUND.value == "PYTHON_NOT_FOUND"
assert ErrorCode.VERSION_MISMATCH.value == "VERSION_MISMATCH"
assert ErrorCode.BBT_EXPORT_NOT_FOUND.value == "BBT_EXPORT_NOT_FOUND"
assert ErrorCode.OCR_TOKEN_MISSING.value == "OCR_TOKEN_MISSING"
assert ErrorCode.SYNC_FAILED.value == "SYNC_FAILED"
assert ErrorCode.VALIDATION_ERROR.value == "VALIDATION_ERROR"
assert ErrorCode.INTERNAL_ERROR.value == "INTERNAL_ERROR"
assert ErrorCode.UNKNOWN.value == "UNKNOWN"
def test_str_returns_value(self) -> None:
assert str(ErrorCode.PYTHON_NOT_FOUND) == "PYTHON_NOT_FOUND"
assert str(ErrorCode.UNKNOWN) == "UNKNOWN"
def test_member_count(self) -> None:
assert len(ErrorCode) == 8
def test_is_str_enum(self) -> None:
assert isinstance(ErrorCode.PYTHON_NOT_FOUND, str)

View file

@ -0,0 +1,105 @@
"""Unit tests for PFResult / PFError round-trip serialization."""
from __future__ import annotations
import json
from paperforge.core.errors import ErrorCode
from paperforge.core.result import PFError, PFResult
class TestPFResultBool:
"""PFResult truthiness follows ok field."""
def test_ok_is_true(self) -> None:
result = PFResult(ok=True, command="sync", version="1.0.0")
assert bool(result) is True
def test_not_ok_is_false(self) -> None:
error = PFError(code=ErrorCode.SYNC_FAILED, message="sync failed")
result = PFResult(ok=False, command="sync", version="1.0.0", error=error)
assert bool(result) is False
class TestPFResultRoundTrip:
"""PFResult serialization round-trip: to_dict -> from_dict."""
def test_ok_with_data(self) -> None:
original = PFResult(
ok=True,
command="sync",
version="1.4.17rc3",
data={"created": 5, "updated": 2},
)
d = original.to_dict()
reconstructed = PFResult.from_dict(d)
assert reconstructed.ok == original.ok
assert reconstructed.command == original.command
assert reconstructed.version == original.version
assert reconstructed.data == original.data
assert reconstructed.error is None
def test_with_error(self) -> None:
error = PFError(
code=ErrorCode.SYNC_FAILED,
message="Export file not found",
details={"path": "/tmp/library.json"},
)
original = PFResult(
ok=False,
command="sync",
version="1.4.17rc3",
data=None,
error=error,
)
d = original.to_dict()
reconstructed = PFResult.from_dict(d)
assert reconstructed.ok is False
assert reconstructed.command == "sync"
assert reconstructed.data is None
assert reconstructed.error is not None
assert reconstructed.error.code == ErrorCode.SYNC_FAILED
assert reconstructed.error.message == "Export file not found"
assert reconstructed.error.details == {"path": "/tmp/library.json"}
def test_none_data_serializes_as_null(self) -> None:
result = PFResult(ok=True, command="status", version="1.4.17rc3", data=None)
d = result.to_dict()
assert d["data"] is None
assert d["ok"] is True
def test_no_error_serializes_as_null(self) -> None:
result = PFResult(ok=True, command="status", version="1.4.17rc3")
d = result.to_dict()
assert d["error"] is None
def test_to_json_round_trip(self) -> None:
original = PFResult(
ok=True,
command="sync",
version="1.4.17rc3",
data={"count": 42},
)
json_str = original.to_json()
parsed = json.loads(json_str)
reconstructed = PFResult.from_dict(parsed)
assert reconstructed == original
def test_to_json_indent_two(self) -> None:
result = PFResult(ok=True, command="ping", version="1.0.0")
json_str = result.to_json()
lines = json_str.splitlines()
for line in lines:
if line.strip().startswith('"'):
assert line.startswith(" ") or line.startswith("{")
def test_from_dict_missing_error_returns_none(self) -> None:
data = {
"ok": True,
"command": "test",
"version": "1.0.0",
"data": None,
"error": None,
}
result = PFResult.from_dict(data)
assert result.error is None
assert result.ok is True

View file

@ -0,0 +1,223 @@
"""Unit tests for paperforge.core.state — state machine enums and transitions."""
from __future__ import annotations
from paperforge.core.state import (
ALLOWED_TRANSITIONS,
Lifecycle,
OcrStatus,
PdfStatus,
)
class TestOcrStatus:
"""OcrStatus enum values and legacy mapping."""
def test_values_match_expected_strings(self) -> None:
assert OcrStatus.PENDING.value == "pending"
assert OcrStatus.PROCESSING.value == "processing"
assert OcrStatus.DONE.value == "done"
assert OcrStatus.FAILED.value == "failed"
def test_from_legacy_canonical_pending(self) -> None:
assert OcrStatus.from_legacy("pending") is OcrStatus.PENDING
def test_from_legacy_canonical_processing(self) -> None:
assert OcrStatus.from_legacy("processing") is OcrStatus.PROCESSING
def test_from_legacy_canonical_done(self) -> None:
assert OcrStatus.from_legacy("done") is OcrStatus.DONE
def test_from_legacy_canonical_failed(self) -> None:
assert OcrStatus.from_legacy("failed") is OcrStatus.FAILED
def test_from_legacy_queued_maps_to_pending(self) -> None:
assert OcrStatus.from_legacy("queued") is OcrStatus.PENDING
def test_from_legacy_running_maps_to_processing(self) -> None:
assert OcrStatus.from_legacy("running") is OcrStatus.PROCESSING
def test_from_legacy_error_maps_to_failed(self) -> None:
assert OcrStatus.from_legacy("error") is OcrStatus.FAILED
def test_from_legacy_blocked_maps_to_failed(self) -> None:
assert OcrStatus.from_legacy("blocked") is OcrStatus.FAILED
def test_from_legacy_nopdf_maps_to_failed(self) -> None:
assert OcrStatus.from_legacy("nopdf") is OcrStatus.FAILED
def test_from_legacy_done_incomplete_maps_to_done(self) -> None:
assert OcrStatus.from_legacy("done_incomplete") is OcrStatus.DONE
def test_from_legacy_unknown_maps_to_pending(self) -> None:
assert OcrStatus.from_legacy("nonexistent") is OcrStatus.PENDING
def test_from_legacy_case_and_whitespace_insensitive(self) -> None:
assert OcrStatus.from_legacy(" DONE ") is OcrStatus.DONE
assert OcrStatus.from_legacy("Queued") is OcrStatus.PENDING
def test_str_returns_value(self) -> None:
assert str(OcrStatus.PENDING) == "pending"
assert str(OcrStatus.DONE) == "done"
class TestPdfStatus:
"""PdfStatus enum values."""
def test_values_match_expected_strings(self) -> None:
assert PdfStatus.HEALTHY.value == "healthy"
assert PdfStatus.BROKEN.value == "broken"
assert PdfStatus.MISSING.value == "missing"
def test_str_returns_value(self) -> None:
assert str(PdfStatus.HEALTHY) == "healthy"
assert str(PdfStatus.BROKEN) == "broken"
assert str(PdfStatus.MISSING) == "missing"
class TestLifecycle:
"""Lifecycle enum values."""
def test_values_match_expected_strings(self) -> None:
assert Lifecycle.PDF_READY.value == "pdf_ready"
assert Lifecycle.OCR_READY.value == "ocr_ready"
assert Lifecycle.ANALYZE_READY.value == "analyze_ready"
assert Lifecycle.DEEP_READ_DONE.value == "deep_read_done"
assert Lifecycle.ERROR_STATE.value == "error_state"
def test_str_returns_value(self) -> None:
assert str(Lifecycle.PDF_READY) == "pdf_ready"
def test_enum_equals_string_value(self) -> None:
"""str enum members compare equal to their string value for backward compat."""
assert Lifecycle.PDF_READY == "pdf_ready"
assert Lifecycle.DEEP_READ_DONE == "deep_read_done"
class TestAllowedTransitionsOcr:
"""ALLOWED_TRANSITIONS.check_ocr validation."""
def test_pending_to_processing_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.PROCESSING)
assert ok is True
assert msg == ""
def test_pending_to_failed_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.FAILED)
assert ok is True
assert msg == ""
def test_processing_to_done_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PROCESSING, OcrStatus.DONE)
assert ok is True
assert msg == ""
def test_processing_to_failed_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PROCESSING, OcrStatus.FAILED)
assert ok is True
assert msg == ""
def test_done_to_pending_valid_rerun(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.DONE, OcrStatus.PENDING)
assert ok is True
assert msg == ""
def test_failed_to_pending_valid_retry(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.FAILED, OcrStatus.PENDING)
assert ok is True
assert msg == ""
def test_pending_to_done_invalid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.DONE)
assert ok is False
assert "Illegal OCR transition" in msg
def test_done_to_processing_invalid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.DONE, OcrStatus.PROCESSING)
assert ok is False
assert "Illegal OCR transition" in msg
def test_failed_to_done_invalid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.FAILED, OcrStatus.DONE)
assert ok is False
assert "Illegal OCR transition" in msg
def test_invalid_transition_includes_allowed_list(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_ocr(OcrStatus.PENDING, OcrStatus.DONE)
assert ok is False
assert "processing" in msg
assert "failed" in msg
class TestAllowedTransitionsLifecycle:
"""ALLOWED_TRANSITIONS.check_lifecycle validation."""
def test_pdf_ready_to_ocr_ready_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.PDF_READY, Lifecycle.OCR_READY
)
assert ok is True
assert msg == ""
def test_pdf_ready_to_error_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.PDF_READY, Lifecycle.ERROR_STATE
)
assert ok is True
assert msg == ""
def test_ocr_ready_to_analyze_ready_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.OCR_READY, Lifecycle.ANALYZE_READY
)
assert ok is True
assert msg == ""
def test_analyze_ready_to_deep_read_done_valid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.ANALYZE_READY, Lifecycle.DEEP_READ_DONE
)
assert ok is True
assert msg == ""
def test_deep_read_done_to_pdf_ready_valid_rerun(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.DEEP_READ_DONE, Lifecycle.PDF_READY
)
assert ok is True
assert msg == ""
def test_error_state_to_pdf_ready_valid_recover(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.ERROR_STATE, Lifecycle.PDF_READY
)
assert ok is True
assert msg == ""
def test_pdf_ready_to_deep_read_done_invalid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.PDF_READY, Lifecycle.DEEP_READ_DONE
)
assert ok is False
assert "Illegal lifecycle transition" in msg
def test_ocr_ready_to_deep_read_done_invalid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.OCR_READY, Lifecycle.DEEP_READ_DONE
)
assert ok is False
assert "Illegal lifecycle transition" in msg
def test_deep_read_done_to_analyze_ready_invalid(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.DEEP_READ_DONE, Lifecycle.ANALYZE_READY
)
assert ok is False
assert "Illegal lifecycle transition" in msg
def test_invalid_transition_includes_allowed_list(self) -> None:
ok, msg = ALLOWED_TRANSITIONS.check_lifecycle(
Lifecycle.PDF_READY, Lifecycle.DEEP_READ_DONE
)
assert ok is False
assert "ocr_ready" in msg
assert "error_state" in msg

View file

View file

@ -0,0 +1,106 @@
from pathlib import Path
from paperforge.doctor.field_validator import (
validate_entry_fields,
validate_collection,
validate_frontmatter_from_file,
)
SAMPLE_REGISTRY = {
"frontmatter": {
"zotero_key": {"type": "str", "required": True, "public": True},
"domain": {"type": "str", "required": True, "public": True},
"title": {"type": "str", "required": True, "public": True},
"year": {"type": "str", "required": False, "public": True},
"doi": {"type": "str", "required": False, "public": True},
"has_pdf": {"type": "bool", "required": True, "public": True},
"analyze": {"type": "bool", "required": False, "public": True},
"ocr_status": {"type": "str", "required": False, "public": True},
},
}
def test_complete_entry_no_issues():
entry = {
"zotero_key": "ABCDEF",
"domain": "骨科",
"title": "A Study",
"year": "2024",
"doi": "10.1234/test",
"has_pdf": "true",
"analyze": "false",
"ocr_status": "pending",
}
issues = validate_entry_fields(entry, "frontmatter", SAMPLE_REGISTRY)
assert issues == []
def test_missing_required_field():
entry = {
"domain": "骨科",
"title": "A Study",
"has_pdf": "true",
}
issues = validate_entry_fields(entry, "frontmatter", SAMPLE_REGISTRY, "ABCDEF")
required_errors = [i for i in issues if i["code"] == "MISSING_REQUIRED"]
assert len(required_errors) == 1
assert required_errors[0]["severity"] == "error"
assert required_errors[0]["field"] == "zotero_key"
def test_unknown_field_drift_warning():
entry = {
"zotero_key": "ABCDEF",
"domain": "骨科",
"title": "A Study",
"has_pdf": "true",
"unknown_field": "something",
}
issues = validate_entry_fields(entry, "frontmatter", SAMPLE_REGISTRY)
drift = [i for i in issues if i["code"] == "DRIFT"]
assert len(drift) == 1
assert drift[0]["severity"] == "warning"
assert drift[0]["field"] == "unknown_field"
def test_missing_optional_field():
entry = {
"zotero_key": "ABCDEF",
"domain": "骨科",
"title": "A Study",
"has_pdf": "true",
}
issues = validate_entry_fields(entry, "frontmatter", SAMPLE_REGISTRY)
optional_missing = [i for i in issues if i["code"] == "MISSING_OPTIONAL"]
assert len(optional_missing) > 0
assert all(i["severity"] == "info" for i in optional_missing)
def test_validate_collection_mixed():
entries = [
{"zotero_key": "A1", "domain": "骨科", "title": "T1", "has_pdf": "true"},
{"zotero_key": "A2", "domain": "骨科", "has_pdf": "true"},
]
result = validate_collection(entries, "frontmatter", SAMPLE_REGISTRY)
assert result["total_entries"] == 2
assert result["entries_with_errors"] == 1
assert "MISSING_REQUIRED" in result["summary"]
def test_validate_frontmatter_valid(tmp_path):
note = tmp_path / "ABCDEF - Test.md"
note.write_text(
"---\nzotero_key: ABCDEF\ndomain: 骨科\ntitle: Test\nhas_pdf: true\n---\n\nSome content",
encoding="utf-8",
)
issues = validate_frontmatter_from_file(note, SAMPLE_REGISTRY)
required_errors = [i for i in issues if i["severity"] == "error"]
assert len(required_errors) == 0
def test_validate_frontmatter_no_frontmatter(tmp_path):
note = tmp_path / "NoFrontmatter.md"
note.write_text("Just content without frontmatter", encoding="utf-8")
issues = validate_frontmatter_from_file(note, SAMPLE_REGISTRY)
assert len(issues) == 1
assert issues[0]["code"] == "NO_FRONTMATTER"

View file

View file

@ -0,0 +1,90 @@
from __future__ import annotations
from pathlib import Path
from paperforge.schema import (
get_field_info,
get_owner_fields,
load_field_registry,
)
HERE = Path(__file__).resolve().parent
REGISTRY_PATH = HERE.parent.parent.parent / "paperforge" / "schema" / "field_registry.yaml"
class TestLoadFieldRegistry:
def test_valid_path_returns_non_empty(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
assert isinstance(reg, dict)
assert len(reg) > 0
def test_nonexistent_path_returns_empty(self) -> None:
reg = load_field_registry(Path("/nonexistent/path.yaml"))
assert reg == {}
def test_default_path_works(self) -> None:
reg = load_field_registry()
assert isinstance(reg, dict)
assert len(reg) > 0
class TestRegistryOwners:
def test_has_all_three_owners(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
assert "frontmatter" in reg
assert "index_entry" in reg
assert "ocr_meta" in reg
assert len(reg) == 3
class TestFrontmatterFields:
def test_has_required_fields(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
fm = get_owner_fields(reg, "frontmatter")
required = {"zotero_key", "domain", "title", "has_pdf"}
for field in required:
assert field in fm, f"Missing required frontmatter field: {field}"
assert fm[field]["required"] is True, f"{field} should be required"
def test_all_expected_fields_present(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
fm = get_owner_fields(reg, "frontmatter")
expected = {
"zotero_key", "domain", "title", "year", "doi",
"collection_path", "has_pdf", "pdf_path", "supplementary",
"fulltext_md_path", "recommend_analyze", "analyze",
"do_ocr", "ocr_status", "deep_reading_status", "path_error",
}
assert set(fm.keys()) == expected
class TestGetOwnerFields:
def test_returns_correct_owner(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
fm = get_owner_fields(reg, "frontmatter")
assert "zotero_key" in fm
assert fm["zotero_key"]["type"] == "str"
def test_unknown_owner_returns_empty(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
assert get_owner_fields(reg, "nonexistent") == {}
class TestGetFieldInfo:
def test_known_field_returns_metadata(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
info = get_field_info(reg, "frontmatter", "zotero_key")
assert info is not None
assert info["type"] == "str"
assert info["required"] is True
assert info["public"] is True
def test_unknown_field_returns_none(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
info = get_field_info(reg, "frontmatter", "nonexistent_field")
assert info is None
def test_unknown_owner_field_returns_none(self) -> None:
reg = load_field_registry(REGISTRY_PATH)
info = get_field_info(reg, "nonexistent", "zotero_key")
assert info is None

View file