From e9a31705d3da00a0de19680a9c79cce1e9230f09 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Sat, 9 May 2026 22:31:08 +0800 Subject: [PATCH] style: ruff format pass across paperforge/ --- paperforge/adapters/zotero_paths.py | 1 + paperforge/commands/context.py | 7 +- paperforge/commands/dashboard.py | 12 +-- paperforge/doctor/field_validator.py | 56 +++++++---- paperforge/setup/__init__.py | 1 + paperforge/setup/runtime.py | 12 +-- paperforge/setup/vault.py | 4 +- paperforge/setup_wizard.py | 95 +++++++++++++----- paperforge/worker/asset_state.py | 6 +- paperforge/worker/base_views.py | 36 ++++++- paperforge/worker/discussion.py | 12 +-- paperforge/worker/ocr.py | 50 +++++----- paperforge/worker/paper_meta.py | 4 +- paperforge/worker/repair.py | 11 ++- paperforge/worker/status.py | 139 ++++++++++++++++----------- paperforge/worker/update.py | 15 ++- 16 files changed, 284 insertions(+), 177 deletions(-) diff --git a/paperforge/adapters/zotero_paths.py b/paperforge/adapters/zotero_paths.py index e2c5a239..50a4e5fe 100644 --- a/paperforge/adapters/zotero_paths.py +++ b/paperforge/adapters/zotero_paths.py @@ -25,6 +25,7 @@ def obsidian_wikilink_for_pdf(pdf_path: str, vault_dir: Path, zotero_dir: Path | if zotero_dir is not None and zotero_dir.exists(): try: from paperforge.pdf_resolver import resolve_junction + real_zotero = resolve_junction(zotero_dir) if real_zotero != zotero_dir: rel_to_zotero = absolute_path.relative_to(real_zotero) diff --git a/paperforge/commands/context.py b/paperforge/commands/context.py index 6d6cb521..a1bd3edf 100644 --- a/paperforge/commands/context.py +++ b/paperforge/commands/context.py @@ -153,12 +153,7 @@ def run(args: argparse.Namespace) -> int: elif collection: # Collection filter: prefix match on any element in "collections" list filtered = [ - e - for e in items - if any( - isinstance(c, str) and c.startswith(collection) - for c in e.get("collections", []) - ) + e for e in items if any(isinstance(c, str) and c.startswith(collection) for c in e.get("collections", [])) ] elif all_mode: diff --git a/paperforge/commands/dashboard.py b/paperforge/commands/dashboard.py index fc1a1a70..68b0e79d 100644 --- a/paperforge/commands/dashboard.py +++ b/paperforge/commands/dashboard.py @@ -69,9 +69,7 @@ def _gather_dashboard_data(vault: Path) -> dict: if paths["literature"].exists(): for domain_dir in sorted(paths["literature"].iterdir()): if domain_dir.is_dir(): - count = sum( - 1 for p in domain_dir.rglob("*.md") if p.name not in _skip_names - ) + count = sum(1 for p in domain_dir.rglob("*.md") if p.name not in _skip_names) if count > 0: domain_counts[domain_dir.name] = count @@ -85,8 +83,8 @@ def _gather_dashboard_data(vault: Path) -> dict: _path_error_pat = re.compile(r'^path_error:\s*"(.+?)"\s*$', re.MULTILINE) _pdf_path_pat = re.compile(r'^pdf_path:\s*".*?"\s*$', re.MULTILINE) - _ocr_status_pat = re.compile(r'^ocr_status:\s*(\S+)', re.MULTILINE) - _do_ocr_pat = re.compile(r'^do_ocr:\s*true\s*$', re.MULTILINE) + _ocr_status_pat = re.compile(r"^ocr_status:\s*(\S+)", re.MULTILINE) + _do_ocr_pat = re.compile(r"^do_ocr:\s*true\s*$", re.MULTILINE) if paths["literature"].exists(): for note_path in paths["literature"].rglob("*.md"): @@ -126,9 +124,7 @@ def _gather_dashboard_data(vault: Path) -> dict: can_sync = len(export_files) > 0 paddle_token = ( - os.environ.get("PADDLEOCR_API_TOKEN") - or os.environ.get("PADDLEOCR_API_KEY") - or os.environ.get("OCR_TOKEN") + os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN") ) can_ocr = bool(paddle_token) diff --git a/paperforge/doctor/field_validator.py b/paperforge/doctor/field_validator.py index cdc4e542..c4a326eb 100644 --- a/paperforge/doctor/field_validator.py +++ b/paperforge/doctor/field_validator.py @@ -29,32 +29,38 @@ def validate_entry_fields( # Check 1: Required fields present for field_name, meta in owner_fields.items(): if meta.get("required", False) and field_name not in entry: - issues.append({ - "severity": "error", - "code": "MISSING_REQUIRED", - "field": field_name, - "message": f"Missing required field '{field_name}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''}", - "suggestion": f"Add '{field_name}' to the entry with appropriate value ({meta.get('type', 'str')})", - }) + issues.append( + { + "severity": "error", + "code": "MISSING_REQUIRED", + "field": field_name, + "message": f"Missing required field '{field_name}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''}", + "suggestion": f"Add '{field_name}' to the entry with appropriate value ({meta.get('type', 'str')})", + } + ) elif not meta.get("required", True) and field_name not in entry: - issues.append({ - "severity": "info", - "code": "MISSING_OPTIONAL", - "field": field_name, - "message": f"Missing optional field '{field_name}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''}", - }) + issues.append( + { + "severity": "info", + "code": "MISSING_OPTIONAL", + "field": field_name, + "message": f"Missing optional field '{field_name}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''}", + } + ) # Check 2: Unknown fields (drift detection) known_fields = set(owner_fields.keys()) for key in entry: if key not in known_fields: - issues.append({ - "severity": "warning", - "code": "DRIFT", - "field": key, - "message": f"Unknown field '{key}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''} — not in field registry", - "suggestion": f"Either remove '{key}' or add it to field_registry.yaml under '{owner}'", - }) + issues.append( + { + "severity": "warning", + "code": "DRIFT", + "field": key, + "message": f"Unknown field '{key}' in {owner} entry{(' (' + entry_label + ')') if entry_label else ''} — not in field registry", + "suggestion": f"Either remove '{key}' or add it to field_registry.yaml under '{owner}'", + } + ) return issues @@ -143,9 +149,17 @@ def validate_frontmatter_from_file( # Parse frontmatter import re + fm_match = re.match(r"^---\s*\n(.*?)\n---", text, re.DOTALL) if not fm_match: - return [{"severity": "warning", "code": "NO_FRONTMATTER", "field": "", "message": f"No frontmatter found in {file_path.name}"}] + return [ + { + "severity": "warning", + "code": "NO_FRONTMATTER", + "field": "", + "message": f"No frontmatter found in {file_path.name}", + } + ] frontmatter = {} for line in fm_match.group(1).splitlines(): diff --git a/paperforge/setup/__init__.py b/paperforge/setup/__init__.py index c6981f74..de777538 100644 --- a/paperforge/setup/__init__.py +++ b/paperforge/setup/__init__.py @@ -9,6 +9,7 @@ from typing import Any @dataclass class SetupStepResult: """Result of a single setup step.""" + step: str ok: bool = True message: str = "" diff --git a/paperforge/setup/runtime.py b/paperforge/setup/runtime.py index 12277fc7..2c93c01b 100644 --- a/paperforge/setup/runtime.py +++ b/paperforge/setup/runtime.py @@ -31,9 +31,7 @@ class RuntimeInstaller: if self.progress_callback: self.progress_callback(message) - def _pip_install( - self, package_spec: str - ) -> tuple[bool, str, str]: + def _pip_install(self, package_spec: str) -> tuple[bool, str, str]: """Run pip install and return (ok, stdout, stderr).""" try: result = subprocess.run( @@ -53,9 +51,7 @@ class RuntimeInstaller: self._log("Installing PaperForge...") if self.version: - package_spec = ( - f"git+https://github.com/LLLin000/PaperForge.git@{self.version}" - ) + package_spec = f"git+https://github.com/LLLin000/PaperForge.git@{self.version}" else: package_spec = "git+https://github.com/LLLin000/PaperForge.git" @@ -65,9 +61,7 @@ class RuntimeInstaller: return SetupStepResult( step="runtime_installer", ok=True, - message=( - f"PaperForge installed successfully{(' (' + self.version + ')') if self.version else ''}" - ), + message=(f"PaperForge installed successfully{(' (' + self.version + ')') if self.version else ''}"), details={"version": self.version or "latest", "stdout": stdout[:500]}, ) else: diff --git a/paperforge/setup/vault.py b/paperforge/setup/vault.py index 4a0da535..25912122 100644 --- a/paperforge/setup/vault.py +++ b/paperforge/setup/vault.py @@ -55,9 +55,7 @@ class VaultInitializer: def create_zotero_junction(self, zotero_path: str | None = None) -> SetupStepResult: """Create Zotero junction/symlink to vault.""" - system_dir = self.vault / self.config.get( - "system_dir", self.DEFAULT_DIRS["system_dir"] - ) + system_dir = self.vault / self.config.get("system_dir", self.DEFAULT_DIRS["system_dir"]) zotero_link = system_dir / "Zotero" if zotero_link.exists() or zotero_link.is_symlink(): diff --git a/paperforge/setup_wizard.py b/paperforge/setup_wizard.py index b542f9f3..ec5db4e5 100644 --- a/paperforge/setup_wizard.py +++ b/paperforge/setup_wizard.py @@ -499,11 +499,7 @@ def _merge_env_incremental(env_path: Path, values: dict[str, str]) -> str: for line in existing_text.splitlines() if line.strip() and not line.lstrip().startswith("#") and "=" in line } - missing_lines = [ - f"{key}={value}" - for key, value in values.items() - if key not in existing_keys - ] + missing_lines = [f"{key}={value}" for key, value in values.items() if key not in existing_keys] if not missing_lines: return "preserved" @@ -546,7 +542,9 @@ def _deploy_skill_directory( skill_dst = vault / skill_dir / skill_name skill_dst.mkdir(parents=True, exist_ok=True) text = skill_file.read_text(encoding="utf-8") - text = _substitute_vars(text, system_dir, resources_dir, literature_dir, control_dir, base_dir, skill_dir, prefix) + text = _substitute_vars( + text, system_dir, resources_dir, literature_dir, control_dir, base_dir, skill_dir, prefix + ) _write_text_incremental(skill_dst / "SKILL.md", text) imported.append(skill_name) @@ -673,7 +671,7 @@ def headless_setup( # Determine repo_root (where paperforge package sources live) if repo_root is None: wizard_dir = Path(__file__).parent.resolve() - if (wizard_dir / "paperforge" if wizard_dir.name != "paperforge" else False): + if wizard_dir / "paperforge" if wizard_dir.name != "paperforge" else False: _repo = wizard_dir elif (wizard_dir.parent / "paperforge").exists(): _repo = wizard_dir.parent @@ -749,7 +747,9 @@ def headless_setup( if sys.platform == "win32": result = subprocess.run( ["cmd", "/c", "mklink", "/J", str(zotero_link_path), str(zotero_data)], - capture_output=True, text=True, timeout=30, + capture_output=True, + text=True, + timeout=30, ) if result.returncode != 0: print(f" [WARN] Zotero junction failed: {result.stderr.strip()}") @@ -837,9 +837,18 @@ def headless_setup( return 4 _copy_file_incremental(worker_src, worker_dst) - for mod in ["ocr.py", "repair.py", "status.py", "deep_reading.py", - "update.py", "base_views.py", "__init__.py", - "_utils.py", "_progress.py", "_retry.py"]: + for mod in [ + "ocr.py", + "repair.py", + "status.py", + "deep_reading.py", + "update.py", + "base_views.py", + "__init__.py", + "_utils.py", + "_progress.py", + "_retry.py", + ]: mod_src = repo_root / "paperforge/worker" / mod if mod_src.exists(): _copy_file_incremental(mod_src, pf_path / "worker/scripts" / mod) @@ -852,24 +861,52 @@ def headless_setup( if fmt == "flat_command": imported_skills = _deploy_flat_command( - vault, agent_config["command_dir"], repo_root, - system_dir, resources_dir, literature_dir, control_dir, base_dir, skill_dir, + vault, + agent_config["command_dir"], + repo_root, + system_dir, + resources_dir, + literature_dir, + control_dir, + base_dir, + skill_dir, ) # OpenCode also needs the skill directory (ld_deep.py, prompt, chart-reading) imported_skills += _deploy_skill_directory( - vault, skill_dir, repo_root, - system_dir, resources_dir, literature_dir, control_dir, base_dir, prefix, + vault, + skill_dir, + repo_root, + system_dir, + resources_dir, + literature_dir, + control_dir, + base_dir, + prefix, ) elif fmt == "rules_file": imported_skills = _deploy_rules_file( - vault, agent_config["skill_dir"], repo_root, - system_dir, resources_dir, literature_dir, control_dir, base_dir, skill_dir, + vault, + agent_config["skill_dir"], + repo_root, + system_dir, + resources_dir, + literature_dir, + control_dir, + base_dir, + skill_dir, ) else: # skill_directory (default) imported_skills = _deploy_skill_directory( - vault, skill_dir, repo_root, - system_dir, resources_dir, literature_dir, control_dir, base_dir, prefix, + vault, + skill_dir, + repo_root, + system_dir, + resources_dir, + literature_dir, + control_dir, + base_dir, + prefix, ) if imported_skills: @@ -997,26 +1034,32 @@ def headless_setup( try: try: import paperforge as _pf - current_ver = getattr(_pf, '__version__', '?') + + current_ver = getattr(_pf, "__version__", "?") except ImportError: - current_ver = 'not installed' + current_ver = "not installed" # If repo_root is the source repository (has pyproject.toml), install from it. # Otherwise (site-packages copy) install from GitHub tagged release. if (repo_root / "pyproject.toml").exists(): install_target = [str(repo_root)] else: from paperforge import __version__ as _pv + install_target = [f"git+https://github.com/LLLin000/PaperForge.git@{_pv}"] result = subprocess.run( [sys.executable, "-m", "pip", "install", "--upgrade"] + install_target, - capture_output=True, text=True, timeout=120, + capture_output=True, + text=True, + timeout=120, ) if result.returncode == 0: _new = subprocess.run( [sys.executable, "-c", "import paperforge; print(getattr(paperforge, '__version__', '?'))"], - capture_output=True, text=True, timeout=15, + capture_output=True, + text=True, + timeout=15, ) - new_ver = _new.stdout.strip() or '?' + new_ver = _new.stdout.strip() or "?" print(f" [OK] paperforge {current_ver} -> {new_ver}") else: stderr_short = result.stderr[:200] if result.stderr else "" @@ -1057,7 +1100,9 @@ def headless_setup( print(" Existing files in the target vault were preserved; setup only created missing files and folders.") print() print("Next steps:") - print(f" 1. In Zotero, export the library or a collection as Better BibTeX JSON into: {vault / system_dir / 'PaperForge' / 'exports'}") + print( + f" 1. In Zotero, export the library or a collection as Better BibTeX JSON into: {vault / system_dir / 'PaperForge' / 'exports'}" + ) print(" Enable 'Keep updated' so Zotero keeps the JSON in sync.") print(" 2. Open Obsidian → Settings → Community Plugins → Enable 'PaperForge'") print(" 3. Press Ctrl+P and type 'PaperForge' to open the dashboard") diff --git a/paperforge/worker/asset_state.py b/paperforge/worker/asset_state.py index 57603eb3..70b2c616 100644 --- a/paperforge/worker/asset_state.py +++ b/paperforge/worker/asset_state.py @@ -86,11 +86,7 @@ def compute_health(entry: dict) -> dict[str, str]: ocr_health = ocr_messages.get(ocr_status, "OCR pending: run `paperforge ocr`") # Note health - note_health = ( - "Formal note missing: run `paperforge sync` to regenerate" - if not note_path - else "healthy" - ) + note_health = "Formal note missing: run `paperforge sync` to regenerate" if not note_path else "healthy" # Asset health — check three workspace paths (deep reading lives in main note) workspace_paths = { diff --git a/paperforge/worker/base_views.py b/paperforge/worker/base_views.py index 387f17ed..789d35bc 100644 --- a/paperforge/worker/base_views.py +++ b/paperforge/worker/base_views.py @@ -104,17 +104,46 @@ def build_base_views(domain: str) -> list[dict]: }, { "name": "待深度阅读", - "order": ["year", "first_author", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"], + "order": [ + "year", + "first_author", + "title", + "has_pdf", + "do_ocr", + "analyze", + "ocr_status", + "deep_reading_status", + "pdf_path", + ], "filter": "analyze = true AND ocr_status = 'done' AND deep_reading_status = 'pending'", }, { "name": "深度阅读完成", - "order": ["year", "first_author", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"], + "order": [ + "year", + "first_author", + "title", + "has_pdf", + "do_ocr", + "analyze", + "ocr_status", + "deep_reading_status", + "pdf_path", + ], "filter": "deep_reading_status = 'done'", }, { "name": "正式卡片", - "order": ["title", "year", "first_author", "journal", "impact_factor", "has_pdf", "deep_reading_status", "pdf_path"], + "order": [ + "title", + "year", + "first_author", + "journal", + "impact_factor", + "has_pdf", + "deep_reading_status", + "pdf_path", + ], "filter": "deep_reading_status = 'done'", }, { @@ -334,6 +363,7 @@ views: def _update_folder_filter(content: str, new_filter: str) -> str: """Update the folder filter in a .base file if it changed.""" import re + old_match = re.search(r'file\.inFolder\("([^"]+)"\)', content) if not old_match or old_match.group(1) == new_filter: return content diff --git a/paperforge/worker/discussion.py b/paperforge/worker/discussion.py index 0ee0cf1e..10477663 100644 --- a/paperforge/worker/discussion.py +++ b/paperforge/worker/discussion.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) _SCHEMA_VERSION = "1" _ISO_FORMAT = "%Y-%m-%dT%H:%M:%S%z" _MD_SEPARATOR = "---" -_MD_SPECIAL_CHARS = re.compile(r'([*#\[\]_`])') +_MD_SPECIAL_CHARS = re.compile(r"([*#\[\]_`])") """Regex for markdown special characters that must be escaped in QA text fields.""" LOCK_TIMEOUT = 10 @@ -63,7 +63,7 @@ def _escape_md(text: str) -> str: when user questions or AI answers contain these characters. Does not double-escape already-escaped characters. """ - return _MD_SPECIAL_CHARS.sub(r'\\\1', text) + return _MD_SPECIAL_CHARS.sub(r"\\\1", text) def _today_str(iso_stamp: str) -> str: @@ -333,12 +333,10 @@ def record_session( _atomic_write_md(md_path, content) except filelock.Timeout: logger.warning("Could not acquire lock for discussion files: %s", lock_path) - return {"status": "error", - "message": "Concurrent access conflict. Please try again."} + return {"status": "error", "message": "Concurrent access conflict. Please try again."} except Exception as exc: logger.warning("Failed to write discussion files: %s", exc) - return {"status": "error", - "message": f"Failed to write discussion files: {exc}"} + return {"status": "error", "message": f"Failed to write discussion files: {exc}"} return { "status": "ok", @@ -368,7 +366,7 @@ def _build_cli_parser() -> argparse.ArgumentParser: record_p.add_argument( "--qa-pairs", default="[]", - help='JSON array of Q&A pairs', + help="JSON array of Q&A pairs", ) return parser diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index a800b842..0b10c7c9 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -33,6 +33,8 @@ def _read_dotenv(vault: Path, key: str) -> str: except (OSError, UnicodeDecodeError): continue return "" + + from paperforge.worker.asset_index import refresh_index_entry from paperforge.worker._retry import retry_with_meta from paperforge.worker._utils import ( @@ -282,11 +284,7 @@ def normalize_obsidian_markdown(text: str) -> str: ) normalized = re.sub( r"(? in for line in lines: page_payload = json.loads(line)["result"] all_results.append(page_payload) - page_num, markdown_path, json_path, fulltext_md_path = postprocess_ocr_result(vault, key, all_results) + page_num, markdown_path, json_path, fulltext_md_path = postprocess_ocr_result( + vault, key, all_results + ) except Exception as e: meta["ocr_status"] = "pending" meta["error"] = str(e) @@ -1725,21 +1725,23 @@ def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False) -> in active_submitted = max(0, active_submitted - 1) write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 + # Upload pending items in batches until none remain (processes all do_ocr items, not just max_items) def _do_upload(token_val: str, pdf_path: Path) -> requests.Response: - with open(pdf_path, "rb") as file_handle: - resp = requests.post( - job_url, - headers={"Authorization": f"bearer {token_val}"}, - data={"model": model, "optionalPayload": json.dumps(optional_payload)}, - files={"file": file_handle}, - timeout=120, - ) - resp.raise_for_status() - return resp + with open(pdf_path, "rb") as file_handle: + resp = requests.post( + job_url, + headers={"Authorization": f"bearer {token_val}"}, + data={"model": model, "optionalPayload": json.dumps(optional_payload)}, + files={"file": file_handle}, + timeout=120, + ) + resp.raise_for_status() + return resp # Combined upload + poll loop: process all items in batches up to max_items concurrency import time as _time + poll_interval = int(os.environ.get("PAPERFORGE_POLL_INTERVAL", "15")) max_poll_cycles = int(os.environ.get("PAPERFORGE_POLL_MAX_CYCLES", "60")) for _cycle in range(max_poll_cycles): @@ -1748,11 +1750,9 @@ def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False) -> in break available_slots = max(0, max_items - active_submitted) if available_slots > 0: - upload_items = [ - r - for r in remaining - if r.get("queue_status", "") not in ("queued", "running") - ][:available_slots] + upload_items = [r for r in remaining if r.get("queue_status", "") not in ("queued", "running")][ + :available_slots + ] for queue_row in upload_items: key = queue_row["zotero_key"] meta = ensure_ocr_meta(vault, queue_row) @@ -1782,6 +1782,7 @@ def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False) -> in if meta.get("needs_sanitize"): try: import tempfile + doc = fitz.open(str(resolved_pdf)) _sanitized_temp = Path(tempfile.mktemp(suffix=".pdf")) doc.save(str(_sanitized_temp), garbage=4, deflate=True, clean=True) @@ -1804,8 +1805,9 @@ def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False) -> in if _sanitized_temp is not None and _sanitized_temp.exists(): _sanitized_temp.unlink(missing_ok=True) import requests as _requests + if isinstance(e, _requests.exceptions.HTTPError): - _status = getattr(getattr(e, 'response', None), 'status_code', 0) + _status = getattr(getattr(e, "response", None), "status_code", 0) if _status == 401: meta["ocr_status"] = "blocked" meta["error"] = "PaddleOCR token invalid" @@ -1896,9 +1898,7 @@ def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False) -> in _time.sleep(poll_interval) # Collect completed OCR keys for incremental index refresh (before filtering) _done_ocr_keys = ( - [r.get("zotero_key", "") for r in ocr_queue if r.get("queue_status") == "done"] - if queue_changed - else [] + [r.get("zotero_key", "") for r in ocr_queue if r.get("queue_status") == "done"] if queue_changed else [] ) if queue_changed: ocr_queue = [row for row in ocr_queue if str(row.get("queue_status", "")).lower() != "done"] diff --git a/paperforge/worker/paper_meta.py b/paperforge/worker/paper_meta.py index f58799f2..039b35a3 100644 --- a/paperforge/worker/paper_meta.py +++ b/paperforge/worker/paper_meta.py @@ -69,9 +69,7 @@ def write_paper_meta( elif "migrated_from" in existing: meta["migrated_from"] = existing["migrated_from"] - meta_path.write_text( - json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8" - ) + meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8") return meta_path diff --git a/paperforge/worker/repair.py b/paperforge/worker/repair.py index ad3b1e46..c57dea33 100644 --- a/paperforge/worker/repair.py +++ b/paperforge/worker/repair.py @@ -191,7 +191,11 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals dict with scanned, divergent, fixed, errors counts """ result = {"scanned": 0, "divergent": [], "fixed": 0, "errors": [], "rebuilt": 0} - record_paths = [p for p in paths["literature"].rglob("*.md") if p.name not in ("fulltext.md", "deep-reading.md", "discussion.md")] + record_paths = [ + p + for p in paths["literature"].rglob("*.md") + if p.name not in ("fulltext.md", "deep-reading.md", "discussion.md") + ] for record_path in record_paths: try: record_text = record_path.read_text(encoding="utf-8") @@ -256,10 +260,7 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals meta_ocr_status is not None and meta_validated_status is not None and note_ocr_status != meta_validated_status - and not ( - note_ocr_status == "pending" - and meta_validated_status == "pending" - ) + and not (note_ocr_status == "pending" and meta_validated_status == "pending") ): is_divergent = True div_reason = f"formal_note={note_ocr_status} vs meta post-validation={meta_validated_status}" diff --git a/paperforge/worker/status.py b/paperforge/worker/status.py index ced9d656..58037057 100644 --- a/paperforge/worker/status.py +++ b/paperforge/worker/status.py @@ -216,7 +216,7 @@ _MODULE_MANIFEST = [ def _read_plugin_data(vault: Path) -> dict: """Read Obsidian plugin's data.json to get the user's python_path override. - + Returns empty dict if file not found, invalid, or not a dict. """ plugin_data_path = vault / ".obsidian" / "plugins" / "paperforge" / "data.json" @@ -233,13 +233,13 @@ def _read_plugin_data(vault: Path) -> dict: def _resolve_plugin_interpreter(vault: Path, plugin_data: dict) -> tuple[str, str, list[str]]: """Replicate the plugin's resolvePythonExecutable() logic in pure Python. - + Detection order: 1. Manual override: plugin_data["python_path"] if set and on disk 2. Venv candidates: .paperforge-test-venv, .venv, venv (Scripts/ on Windows, bin/ on POSIX) 3. System candidates: py -3, python, python3 (tested via --version) 4. Fallback: python - + Returns (interpreter_path, source, extra_args). """ # 1. Manual override @@ -288,11 +288,9 @@ def _resolve_plugin_interpreter(vault: Path, plugin_data: dict) -> tuple[str, st return ("python", "auto-detected", []) -def _query_resolved_version( - interp: str, extra_args: list[str] -) -> tuple[str | None, tuple[int, int, int] | None]: +def _query_resolved_version(interp: str, extra_args: list[str]) -> tuple[str | None, tuple[int, int, int] | None]: """Run interpreter --version, parse and return version info. - + Returns (version_string, (major, minor, micro)) or (None, None) on failure. """ try: @@ -312,11 +310,9 @@ def _query_resolved_version( return (None, None) -def _query_resolved_package( - interp: str, extra_args: list[str], package_name: str -) -> dict | None: +def _query_resolved_package(interp: str, extra_args: list[str], package_name: str) -> dict | None: """Run pip show for a package under the resolved interpreter. - + Returns dict with keys Name, Version, Location, etc., or None if not found. """ try: @@ -390,20 +386,24 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> ) # --- Plugin-resolved interpreter checks --- - add_check("Python 环境 (插件)", "pass", - f"已解析解释器: {interp} (来源: {source})") + add_check("Python 环境 (插件)", "pass", f"已解析解释器: {interp} (来源: {source})") if version_tuple is not None and version_tuple >= (3, 10, 0): - add_check("Python 环境 (插件)", "pass", - f"Python {version_str}") + add_check("Python 环境 (插件)", "pass", f"Python {version_str}") elif version_tuple is not None: - add_check("Python 环境 (插件)", "fail", + add_check( + "Python 环境 (插件)", + "fail", f"Python {version_str} (需要 3.10+)", - f"升级解释器 {interp} 到 Python 3.10 或更高版本") + f"升级解释器 {interp} 到 Python 3.10 或更高版本", + ) else: - add_check("Python 环境 (插件)", "fail", + add_check( + "Python 环境 (插件)", + "fail", f"无法获取 {interp} 的 Python 版本", - f"验证 {interp} 是一个有效的 Python 解释器") + f"验证 {interp} 是一个有效的 Python 解释器", + ) resolved_pf_module = _query_resolved_module(interp, extra_args, "paperforge") @@ -412,17 +412,22 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> pkg_location = pf_pkg_info.get("Location", "?") expected_version = __import__("paperforge").__version__ if pkg_version == expected_version: - add_check("PaperForge 包", "pass", - f"v{pkg_version} 已安装 -> {pkg_location}") + add_check("PaperForge 包", "pass", f"v{pkg_version} 已安装 -> {pkg_location}") else: - add_check("PaperForge 包", "warn", + add_check( + "PaperForge 包", + "warn", f"v{pkg_version} 已安装 (插件版本 v{expected_version}) - 版本不匹配", - f"运行: {interp} -m pip install --upgrade git+https://github.com/LLLin000/PaperForge.git@{expected_version}") + f"运行: {interp} -m pip install --upgrade git+https://github.com/LLLin000/PaperForge.git@{expected_version}", + ) else: expected_version = __import__("paperforge").__version__ - add_check("PaperForge 包", "fail", + add_check( + "PaperForge 包", + "fail", f"PaperForge 未安装在 {interp} 中", - f"运行: {interp} -m pip install --upgrade git+https://github.com/LLLin000/PaperForge.git@{expected_version}") + f"运行: {interp} -m pip install --upgrade git+https://github.com/LLLin000/PaperForge.git@{expected_version}", + ) # Wrong-environment detection current_package_file = os.path.normcase(os.path.abspath(__import__("paperforge").__file__)) @@ -430,18 +435,21 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> if resolved_pf_module and resolved_pf_module.get("file"): resolved_package_file = os.path.normcase(os.path.abspath(str(resolved_pf_module["file"]))) if resolved_package_file and resolved_package_file != current_package_file: - add_check("PaperForge 包", "warn", - f"包路径不一致: 已解析解释器 -> {resolved_package_file} | 当前诊断进程 -> {current_package_file}", - "建议: 使用已解析解释器运行 doctor,或统一 Python 环境") + add_check( + "PaperForge 包", + "warn", + f"包路径不一致: 已解析解释器 -> {resolved_package_file} | 当前诊断进程 -> {current_package_file}", + "建议: 使用已解析解释器运行 doctor,或统一 Python 环境", + ) # --- Per-module dependency checks (Phase 53: DOCTOR-03) --- for mod_info in _MODULE_MANIFEST: mod_name = mod_info["import"] mod_info_resolved = _query_resolved_module(interp, extra_args, mod_name) if mod_info_resolved is None: - add_check("Python 环境", "fail", - f"{mod_info['label']} 缺失", - f"运行: {interp} -m pip install {mod_info['pip']}") + add_check( + "Python 环境", "fail", f"{mod_info['label']} 缺失", f"运行: {interp} -m pip install {mod_info['pip']}" + ) continue ver = mod_info_resolved.get("version") @@ -452,15 +460,17 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> try: ver_parts = str(ver).split(".") if int(ver_parts[0]) < 6: - add_check("Python 环境", "fail", + add_check( + "Python 环境", + "fail", f"{mod_info['label']} {ver} (需要 >=6.0)", - f"运行: {interp} -m pip install {mod_info['pip']}") + f"运行: {interp} -m pip install {mod_info['pip']}", + ) continue except (ValueError, IndexError): pass - add_check("Python 环境", "pass", - f"{mod_info['label']} 已安装{ver_str}") + add_check("Python 环境", "pass", f"{mod_info['label']} 已安装{ver_str}") if (vault / "paperforge.json").exists(): add_check("Vault 结构", "pass", "paperforge.json 存在") @@ -612,7 +622,11 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> total_issues += 1 add_check( "字段注册表", - "fail" if issue["severity"] == "error" else "warn" if issue["severity"] == "warning" else "pass", + "fail" + if issue["severity"] == "error" + else "warn" + if issue["severity"] == "warning" + else "pass", issue["message"], issue.get("suggestion", ""), ) @@ -692,7 +706,8 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> try: if int(_sv) < 2: add_check( - "Index Health", "warn", + "Index Health", + "warn", f"Legacy index schema v{_sv} -- consider rebuild to v2", "Run `paperforge sync --rebuild-index`", ) @@ -712,7 +727,8 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> continue if _legacy > 0: add_check( - "Index Health", "warn", + "Index Health", + "warn", f"{_legacy} Base file(s) use legacy columns (has_pdf, do_ocr) instead of lifecycle", "Run `paperforge sync` to regenerate Base views", ) @@ -731,7 +747,8 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> continue if _partial > 0: add_check( - "Index Health", "warn", + "Index Health", + "warn", f"{_partial} partial OCR asset(s) found", "Re-run `paperforge ocr` on affected items", ) @@ -739,6 +756,7 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> # WS-05: Workspace integrity checks try: from paperforge.worker.asset_index import read_index as _ws_read_idx + _idx_content = _ws_read_idx(vault) _items = _idx_content.get("items", []) if isinstance(_idx_content, dict) else [] _missing_workspace = 0 @@ -751,6 +769,7 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> if not _key or not _dom: continue from paperforge.worker._utils import slugify_filename + _slug = slugify_filename(_title or _key) _ws_dir = _ws_literature / _dom / f"{_key} - {_slug}" if not _ws_dir.exists(): @@ -762,13 +781,15 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> _missing_fulltext += 1 if _missing_workspace > 0: add_check( - "Index Health", "warn", + "Index Health", + "warn", f"{_missing_workspace} paper(s) missing workspace directories", "Run `paperforge sync` to create workspace folders", ) if _missing_fulltext > 0: add_check( - "Index Health", "warn", + "Index Health", + "warn", f"{_missing_fulltext} paper(s) missing fulltext.md in workspace", "Re-run OCR or run `paperforge sync` to bridge fulltext", ) @@ -783,7 +804,8 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> _stale_count = sum(1 for _ in _lr_dir.rglob("*.md")) if _stale_count > 0: add_check( - "Index Health", "warn", + "Index Health", + "warn", f"{_stale_count} stale record(s) found in control directory", "Review and remove stale files from the control directory", ) @@ -791,10 +813,7 @@ def run_doctor(vault: Path, verbose: bool = False, json_output: bool = False) -> add_check("Index Health", "info", "No canonical index -- run `paperforge sync` to generate") if json_output: - checklist_data = [ - {"category": cat, "status": st, "message": msg, "fix": fx} - for cat, st, msg, fx in checks - ] + checklist_data = [{"category": cat, "status": st, "message": msg, "fix": fx} for cat, st, msg, fx in checks] status_counts: dict[str, int] = {} for _, st, _, _ in checks: status_counts[st] = status_counts.get(st, 0) + 1 @@ -961,7 +980,11 @@ def run_status(vault: Path, verbose: bool = False, json_output: bool = False) -> ensure_base_views(vault, paths, config) export_files = sorted(paths["exports"].glob("*.json")) record_count = ( - sum(1 for p in paths["literature"].rglob("*.md") if p.name not in ("fulltext.md", "deep-reading.md", "discussion.md")) + sum( + 1 + for p in paths["literature"].rglob("*.md") + if p.name not in ("fulltext.md", "deep-reading.md", "discussion.md") + ) if paths["literature"].exists() else 0 ) @@ -1101,15 +1124,21 @@ def run_status(vault: Path, verbose: bool = False, json_output: bool = False) -> if summary is not None: lc = lifecycle_level_counts print(f"- index: {summary['paper_count']} papers") - print(f" lifecycle: indexed={lc['indexed']} pdf_ready={lc['pdf_ready']} " - f"fulltext_ready={lc['fulltext_ready']} deep_read={lc['deep_read_done']} " - f"ai_ready={lc['ai_context_ready']}") + print( + f" lifecycle: indexed={lc['indexed']} pdf_ready={lc['pdf_ready']} " + f"fulltext_ready={lc['fulltext_ready']} deep_read={lc['deep_read_done']} " + f"ai_ready={lc['ai_context_ready']}" + ) ha = health_aggregate - print(f" health: pdf={ha['pdf_health']['healthy']}/{ha['pdf_health']['unhealthy']} " - f"ocr={ha['ocr_health']['healthy']}/{ha['ocr_health']['unhealthy']} " - f"note={ha['note_health']['healthy']}/{ha['note_health']['unhealthy']} " - f"asset={ha['asset_health']['healthy']}/{ha['asset_health']['unhealthy']}") - print(f"- ocr: {ocr_done}/{ocr_total} done (pending: {ocr_pending}, processing: {ocr_processing}, failed: {ocr_failed})") + print( + f" health: pdf={ha['pdf_health']['healthy']}/{ha['pdf_health']['unhealthy']} " + f"ocr={ha['ocr_health']['healthy']}/{ha['ocr_health']['unhealthy']} " + f"note={ha['note_health']['healthy']}/{ha['note_health']['unhealthy']} " + f"asset={ha['asset_health']['healthy']}/{ha['asset_health']['unhealthy']}" + ) + print( + f"- ocr: {ocr_done}/{ocr_total} done (pending: {ocr_pending}, processing: {ocr_processing}, failed: {ocr_failed})" + ) print(f"- path_errors: {path_error_count}") if path_error_count > 0: print(" Tip: Run `paperforge repair --fix-paths` to attempt resolution") diff --git a/paperforge/worker/update.py b/paperforge/worker/update.py index 0572841d..b3385123 100644 --- a/paperforge/worker/update.py +++ b/paperforge/worker/update.py @@ -52,6 +52,7 @@ def protected_paths(vault: Path) -> set[str]: def _remote_version() -> str | None: """Read version from __init__.py on GitHub (single source of truth).""" import re + try: api = f"https://api.github.com/repos/{GITHUB_REPO}/contents/paperforge/__init__.py" req = urllib.request.Request( @@ -176,12 +177,21 @@ def _update_via_git(vault: Path) -> bool: if not (vault / ".git").is_dir(): logger.error("不是 git 仓库") return False - r = subprocess.run(["git", "status", "--short"], cwd=vault, capture_output=True, text=True, encoding="utf-8", errors="replace") + r = subprocess.run( + ["git", "status", "--short"], cwd=vault, capture_output=True, text=True, encoding="utf-8", errors="replace" + ) if r.stdout.strip(): logger.warning("有未提交的更改,请先提交或储藏") return False logger.info("执行 git pull...") - r = subprocess.run(["git", "pull", "origin", "master"], cwd=vault, capture_output=True, text=True, encoding="utf-8", errors="replace") + r = subprocess.run( + ["git", "pull", "origin", "master"], + cwd=vault, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) if r.returncode != 0: logger.error("git pull 失败: %s", r.stderr) return False @@ -231,6 +241,7 @@ def run_update(vault: Path) -> int: """运行更新检查与安装""" try: import paperforge + local = paperforge.__version__ except Exception: local = "unknown"