diff --git a/paperforge/ocr_diagnostics.py b/paperforge/ocr_diagnostics.py index 57520745..2824681e 100644 --- a/paperforge/ocr_diagnostics.py +++ b/paperforge/ocr_diagnostics.py @@ -167,7 +167,7 @@ def ocr_doctor(config: dict[str, str] | None, live: bool = False) -> dict: return { "level": 4, "passed": False, - "error": f'Live PDF test failed: {poll_data.get("errorMsg", "unknown")}', + "error": f"Live PDF test failed: {poll_data.get('errorMsg', 'unknown')}", "fix": "PDF may be unreadable or OCR service error. Try a different PDF or check service status.", } else: diff --git a/paperforge/worker/base_views.py b/paperforge/worker/base_views.py index a9bef92b..7550e696 100644 --- a/paperforge/worker/base_views.py +++ b/paperforge/worker/base_views.py @@ -1,41 +1,27 @@ from __future__ import annotations + import logging -import argparse -import csv -import hashlib -import html -import json import os -import re -import shutil -import subprocess -import sys -import tempfile -import urllib.parse -from json import JSONDecodeError -import zipfile -from datetime import datetime, timezone from pathlib import Path -from xml.etree import ElementTree as ET -import requests -import fitz -from PIL import Image + +from paperforge.config import paperforge_paths +from paperforge.worker._utils import ( + read_json, + slugify_filename, + write_json, +) logger = logging.getLogger(__name__) -STANDARD_VIEW_NAMES = frozenset([ - "控制面板", "推荐分析", "待 OCR", "OCR 完成", - "待深度阅读", "深度阅读完成", "正式卡片", "全记录" -]) def load_simple_env(env_path: Path) -> None: if not env_path.exists(): return - for raw_line in env_path.read_text(encoding='utf-8').splitlines(): + for raw_line in env_path.read_text(encoding="utf-8").splitlines(): line = raw_line.strip() - if not line or line.startswith('#') or '=' not in line: + if not line or line.startswith("#") or "=" not in line: continue - key, value = line.split('=', 1) + key, value = line.split("=", 1) key = key.strip() if not key or key in os.environ: continue @@ -44,104 +30,6 @@ def load_simple_env(env_path: Path) -> None: value = value[1:-1] os.environ[key] = value -def read_json(path: Path): - return json.loads(path.read_text(encoding='utf-8')) -_JOURNAL_DB: dict[str, dict] | None = None - -def load_journal_db(vault: Path) -> dict[str, dict]: - """Load zoterostyle.json journal database.""" - global _JOURNAL_DB - if _JOURNAL_DB is not None: - return _JOURNAL_DB - zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json' - if zoterostyle_path.exists(): - try: - _JOURNAL_DB = read_json(zoterostyle_path) - except (JSONDecodeError, Exception): - _JOURNAL_DB = {} - else: - _JOURNAL_DB = {} - return _JOURNAL_DB - -def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str: - """Lookup impact factor: prefer zoterostyle.json, fallback to extra field.""" - if not journal_name: - return '' - journal_db = load_journal_db(vault) - if journal_name in journal_db: - rank_data = journal_db[journal_name].get('rank', {}) - if isinstance(rank_data, dict): - sciif = rank_data.get('sciif', '') - if sciif: - return str(sciif) - if extra: - if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra) - if if_match: - return if_match.group(1) - return '' - -def write_json(path: Path, data) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') - -def read_jsonl(path: Path): - rows = [] - if not path.exists(): - return rows - for line in path.read_text(encoding='utf-8').splitlines(): - line = line.strip() - if line: - rows.append(json.loads(line)) - return rows - -def write_jsonl(path: Path, rows) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows)) - if text: - text += '\n' - path.write_text(text, encoding='utf-8') - -def yaml_quote(value: str) -> str: - if isinstance(value, bool): - return 'true' if value else 'false' - return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"' - -def yaml_block(value: str) -> list[str]: - value = (value or '').strip() - if not value: - return ['abstract: |-', ' '] - lines = ['abstract: |-'] - for line in value.splitlines(): - lines.append(f' {line}') - return lines - -def yaml_list(key: str, values) -> list[str]: - cleaned = [str(value).strip() for value in values or [] if str(value).strip()] - if not cleaned: - return [f'{key}: []'] - lines = [f'{key}:'] - for value in cleaned: - lines.append(f' - {yaml_quote(value)}') - return lines - -def slugify_filename(text: str) -> str: - cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip() - return cleaned[:120] or 'untitled' - -def _extract_year(value: str) -> str: - match = re.search('(19|20)\\d{2}', value or '') - return match.group(0) if match else '' - - -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -149,14 +37,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -183,17 +64,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -204,14 +83,17 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + def base_markdown_filter(path: Path, vault: Path) -> str: try: - return str(path.relative_to(vault)).replace('\\', '/') + return str(path.relative_to(vault)).replace("\\", "/") except ValueError: - return str(path).replace('\\', '/') + return str(path).replace("\\", "/") + PAPERFORGE_VIEW_PREFIX = "# PAPERFORGE_VIEW: " + def build_base_views(domain: str) -> list[dict]: """Build the 8-view list for a domain Base file. @@ -224,12 +106,33 @@ def build_base_views(domain: str) -> list[dict]: return [ { "name": "控制面板", - "order": ["file.name", "title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"], + "order": [ + "file.name", + "title", + "year", + "has_pdf", + "do_ocr", + "analyze", + "ocr_status", + "deep_reading_status", + "pdf_path", + "fulltext_md_path", + ], "filter": None, }, { "name": "推荐分析", - "order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"], + "order": [ + "year", + "title", + "has_pdf", + "do_ocr", + "analyze", + "ocr_status", + "deep_reading_status", + "pdf_path", + "fulltext_md_path", + ], "filter": "analyze = true AND recommend_analyze = true", }, { @@ -259,11 +162,22 @@ def build_base_views(domain: str) -> list[dict]: }, { "name": "全记录", - "order": ["title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"], + "order": [ + "title", + "year", + "has_pdf", + "do_ocr", + "analyze", + "ocr_status", + "deep_reading_status", + "pdf_path", + "fulltext_md_path", + ], "filter": None, }, ] + def substitute_config_placeholders(content: str, paths: dict[str, Path]) -> str: """Replace ${SCREAMING_SNAKE_CASE} path placeholders with vault-relative paths. @@ -297,13 +211,13 @@ def _render_views_section(views: list[dict]) -> str: """Render a list of view dicts to YAML views: section.""" lines = [] for v in views: - lines.append(f" - type: table") + lines.append(" - type: table") lines.append(f' name: "{v["name"]}"') - lines.append(f" order:") + lines.append(" order:") for col in v["order"]: lines.append(f" - {col}") if v["filter"]: - lines.append(f' filter: \'{v["filter"]}\'') + lines.append(f" filter: '{v['filter']}'") return "\n".join(lines) @@ -351,7 +265,7 @@ def merge_base_views(existing_content: str | None, new_views: list[dict]) -> str fresh_views_yaml = _render_views_section(new_views) return f"""filters: and: - - file.inFolder("{new_views[0]['name']}") + - file.inFolder("{new_views[0]["name"]}") {PROPERTIES_YAML} views: {fresh_views_yaml}""" @@ -372,20 +286,20 @@ views: views: {fresh_views_yaml}""" - header_lines = lines[:views_start_idx + 1] + header_lines = lines[: views_start_idx + 1] new_pf_blocks = [] for v in new_views: rendered = f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n" - rendered += f" - type: table\n" + rendered += " - type: table\n" rendered += f' name: "{v["name"]}"\n' - rendered += f" order:\n" + rendered += " order:\n" for col in v["order"]: rendered += f" - {col}\n" if v["filter"]: - rendered += f' filter: \'{v["filter"]}\'\n' + rendered += f" filter: '{v['filter']}'\n" else: - rendered += '\n' + rendered += "\n" new_pf_blocks.append((v["name"], rendered)) rebuilt_views_lines = [] @@ -396,7 +310,7 @@ views: line = lines[i] if line.startswith(PAPERFORGE_VIEW_PREFIX): - pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX):].strip() + pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX) :].strip() pf_names_seen.add(pending_pf_view_name) i += 1 continue @@ -440,16 +354,16 @@ def _build_base_yaml(folder_filter: str, views: list[dict]) -> str: views_yaml = "" for v in views: views_yaml += f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n" - views_yaml += f" - type: table\n" + views_yaml += " - type: table\n" views_yaml += f' name: "{v["name"]}"\n' - views_yaml += f" order:\n" + views_yaml += " order:\n" for col in v["order"]: views_yaml += f" - {col}\n" if v["filter"]: - views_yaml += f' filter: \'{v["filter"]}\'\n' + views_yaml += f" filter: '{v['filter']}'\n" else: - views_yaml += '\n' - views_yaml = views_yaml.rstrip('\n') + views_yaml += "\n" + views_yaml = views_yaml.rstrip("\n") return f"""filters: and: - file.inFolder("{folder_filter}") @@ -515,5 +429,3 @@ def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: all_views = build_base_views("All Records") pf_base = paths["bases"] / "PaperForge.base" refresh_base(pf_base, "${LIBRARY_RECORDS}", all_views) - - diff --git a/paperforge/worker/deep_reading.py b/paperforge/worker/deep_reading.py index 2b783e7a..c011a6c9 100644 --- a/paperforge/worker/deep_reading.py +++ b/paperforge/worker/deep_reading.py @@ -1,59 +1,21 @@ from __future__ import annotations + import logging -import argparse -import csv -import hashlib -import html -import json -import os import re -import shutil -import subprocess -import sys -import tempfile -import urllib.parse -from json import JSONDecodeError -import zipfile -from datetime import datetime, timezone from pathlib import Path -from xml.etree import ElementTree as ET -import requests -import fitz -from PIL import Image - -from paperforge.worker.sync import has_deep_reading_content, load_export_rows -from paperforge.worker.base_views import ensure_base_views -from paperforge.worker.ocr import validate_ocr_meta +from paperforge.config import paperforge_paths from paperforge.worker._utils import ( - STANDARD_VIEW_NAMES, - _extract_year, - _JOURNAL_DB, - load_journal_db, - lookup_impact_factor, read_json, - read_jsonl, - slugify_filename, - write_json, - write_jsonl, - yaml_block, - yaml_list, - yaml_quote, scan_library_records, - _resolve_formal_note_path, + write_json, + yaml_quote, ) +from paperforge.worker.base_views import ensure_base_views +from paperforge.worker.sync import has_deep_reading_content logger = logging.getLogger(__name__) -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -61,14 +23,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -95,17 +50,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -116,8 +69,10 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + # Re-exported from _utils.py for backward compatibility + def run_deep_reading(vault: Path, verbose: bool = False) -> int: """Sync deep-reading status between formal notes and library records. @@ -131,83 +86,104 @@ def run_deep_reading(vault: Path, verbose: bool = False) -> int: paths = pipeline_paths(vault) config = load_domain_config(paths) ensure_base_views(vault, paths, config) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} + {entry["export_file"]: entry["domain"] for entry in config["domains"]} synced = 0 pending_queue: list[dict] = [] records = scan_library_records(vault) for record in records: - key = record['zotero_key'] - domain = record['domain'] + key = record["zotero_key"] + domain = record["domain"] # Status sync: check actual note content vs frontmatter status - note_path = record['note_path'] + note_path = record["note_path"] has_content = False if note_path and note_path.exists(): - note_text = note_path.read_text(encoding='utf-8') + note_text = note_path.read_text(encoding="utf-8") has_content = has_deep_reading_content(note_text) - correct_status = 'done' if has_content else 'pending' + correct_status = "done" if has_content else "pending" - if record['deep_reading_status'] != correct_status: - record_dir = paths['library_records'] / domain - record_path = record_dir / f'{key}.md' - record_text = record_path.read_text(encoding='utf-8') + if record["deep_reading_status"] != correct_status: + record_dir = paths["library_records"] / domain + record_path = record_dir / f"{key}.md" + record_text = record_path.read_text(encoding="utf-8") new_text = re.sub( '^deep_reading_status:\\s*"?.*?"?$', - f'deep_reading_status: {yaml_quote(correct_status)}', - record_text, flags=re.MULTILINE, count=1 + f"deep_reading_status: {yaml_quote(correct_status)}", + record_text, + flags=re.MULTILINE, + count=1, ) - record_path.write_text(new_text, encoding='utf-8') + record_path.write_text(new_text, encoding="utf-8") synced += 1 - if correct_status == 'pending': - pending_queue.append({ - 'zotero_key': key, - 'domain': domain, - 'title': record['title'], - 'ocr_status': record['ocr_status'], - 'is_analyze': True, - 'is_do_ocr': record['do_ocr'], - }) + if correct_status == "pending": + pending_queue.append( + { + "zotero_key": key, + "domain": domain, + "title": record["title"], + "ocr_status": record["ocr_status"], + "is_analyze": True, + "is_do_ocr": record["do_ocr"], + } + ) if pending_queue: - ready = [q for q in pending_queue if q['ocr_status'] == 'done'] - waiting = [q for q in pending_queue if q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing')] - blocked = [q for q in pending_queue if q['is_analyze'] and q['ocr_status'] not in ('done', '') and not (q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing'))] - report_lines = ['# 待精读队列', ''] + ready = [q for q in pending_queue if q["ocr_status"] == "done"] + waiting = [q for q in pending_queue if q["is_do_ocr"] and q["ocr_status"] in ("pending", "processing")] + blocked = [ + q + for q in pending_queue + if q["is_analyze"] + and q["ocr_status"] not in ("done", "") + and not (q["is_do_ocr"] and q["ocr_status"] in ("pending", "processing")) + ] + report_lines = ["# 待精读队列", ""] if ready: - report_lines.extend([f'## 就绪 ({len(ready)} 篇) — OCR 已完成,可直接 /pf-deep', '']) + report_lines.extend([f"## 就绪 ({len(ready)} 篇) — OCR 已完成,可直接 /pf-deep", ""]) for q in ready: report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']}") - report_lines.append('') + report_lines.append("") if waiting: - report_lines.extend([f'## 等待 OCR ({len(waiting)} 篇)', '']) + report_lines.extend([f"## 等待 OCR ({len(waiting)} 篇)", ""]) for q in waiting: report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status']}") - report_lines.append('') + report_lines.append("") if blocked: - report_lines.extend([f'## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR', '']) + report_lines.extend([f"## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR", ""]) for q in blocked: - report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}") - report_lines.append('') + report_lines.append( + f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}" + ) + report_lines.append("") if verbose: - report_lines.append('### 修复步骤\n') + report_lines.append("### 修复步骤\n") for q in blocked: - ocr_s = q['ocr_status'] or '' - if not ocr_s or ocr_s == 'pending': - fix = f"paperforge ocr" + ocr_s = q["ocr_status"] or "" + if not ocr_s or ocr_s == "pending": + fix = "paperforge ocr" report_lines.append(f"- `{q['zotero_key']}`: 运行 `{fix}` 启动 OCR") - elif ocr_s == 'processing': + elif ocr_s == "processing": report_lines.append(f"- `{q['zotero_key']}`: OCR 进行中,请等待完成") - elif ocr_s == 'failed': - report_lines.append(f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息,然后重新运行 `paperforge ocr`") + elif ocr_s == "failed": + report_lines.append( + f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息,然后重新运行 `paperforge ocr`" + ) else: report_lines.append(f"- `{q['zotero_key']}`: 运行 `paperforge ocr` 重试") - report_lines.append('') - report_lines.extend(['## 操作', '', '- 对就绪论文,使用 `/pf-deep ` 触发精读', '- 批量触发:提供多个 key,用 subagent 并行处理', '']) + report_lines.append("") + report_lines.extend( + [ + "## 操作", + "", + "- 对就绪论文,使用 `/pf-deep ` 触发精读", + "- 批量触发:提供多个 key,用 subagent 并行处理", + "", + ] + ) else: - report_lines = ['# 待精读队列', '', '所有 analyze=true 的论文已完成精读。', ''] - report_path = paths['pipeline'] / 'deep-reading-queue.md' - report_path.write_text('\n'.join(report_lines), encoding='utf-8') - print(f'deep-reading: synced {synced} records, {len(pending_queue)} pending') + report_lines = ["# 待精读队列", "", "所有 analyze=true 的论文已完成精读。", ""] + report_path = paths["pipeline"] / "deep-reading-queue.md" + report_path.write_text("\n".join(report_lines), encoding="utf-8") + print(f"deep-reading: synced {synced} records, {len(pending_queue)} pending") return 0 - diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index 78254934..497c4270 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -1,62 +1,34 @@ from __future__ import annotations -import logging -import argparse -import csv -import hashlib + import html import json +import logging import os import re import shutil -import subprocess -import sys -import tempfile -import urllib.parse -from json import JSONDecodeError -import zipfile from datetime import datetime, timezone +from json import JSONDecodeError from pathlib import Path -from xml.etree import ElementTree as ET -import requests + import fitz +import requests from PIL import Image +from paperforge.config import paperforge_paths from paperforge.worker import sync as _sync +from paperforge.worker._progress import progress_bar +from paperforge.worker._retry import retry_with_meta +from paperforge.worker._utils import ( + read_json, + write_json, +) from paperforge.worker.sync import ( load_control_actions, load_export_rows, ) -from paperforge.worker._utils import ( - STANDARD_VIEW_NAMES, - _extract_year, - _JOURNAL_DB, - load_journal_db, - lookup_impact_factor, - read_json, - read_jsonl, - slugify_filename, - write_json, - write_jsonl, - yaml_block, - yaml_list, - yaml_quote, -) - -from paperforge.worker._retry import configure_retry, retry_with_meta -from paperforge.worker._progress import progress_bar - logger = logging.getLogger(__name__) -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -64,14 +36,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -98,17 +63,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -119,82 +82,85 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + def ensure_ocr_meta(vault: Path, row: dict) -> dict: paths = pipeline_paths(vault) - key = row['zotero_key'] - meta_path = paths['ocr'] / key / 'meta.json' + key = row["zotero_key"] + meta_path = paths["ocr"] / key / "meta.json" meta_path.parent.mkdir(parents=True, exist_ok=True) meta = read_json(meta_path) if meta_path.exists() else {} - meta.setdefault('zotero_key', key) - meta.setdefault('source_pdf', row.get('pdf_path', '')) - meta.setdefault('ocr_provider', 'PaddleOCR-VL-1.5') - meta.setdefault('mode', 'async') - meta.setdefault('ocr_status', 'pending') - meta.setdefault('ocr_job_id', '') - meta.setdefault('ocr_started_at', '') - meta.setdefault('ocr_finished_at', '') - meta.setdefault('page_count', 0) - meta.setdefault('markdown_path', '') - meta.setdefault('json_path', '') + meta.setdefault("zotero_key", key) + meta.setdefault("source_pdf", row.get("pdf_path", "")) + meta.setdefault("ocr_provider", "PaddleOCR-VL-1.5") + meta.setdefault("mode", "async") + meta.setdefault("ocr_status", "pending") + meta.setdefault("ocr_job_id", "") + meta.setdefault("ocr_started_at", "") + meta.setdefault("ocr_finished_at", "") + meta.setdefault("page_count", 0) + meta.setdefault("markdown_path", "") + meta.setdefault("json_path", "") try: - assets_path = str((paths['ocr'] / key / 'images').relative_to(vault)).replace('\\', '/') + assets_path = str((paths["ocr"] / key / "images").relative_to(vault)).replace("\\", "/") except ValueError: - assets_path = str(paths['ocr'] / key / 'images') - meta.setdefault('assets_path', assets_path) - meta.setdefault('fulltext_md_path', '') - meta.setdefault('error', '') - meta.setdefault('retry_count', 0) - meta.setdefault('last_error', None) - meta.setdefault('last_attempt_at', None) + assets_path = str(paths["ocr"] / key / "images") + meta.setdefault("assets_path", assets_path) + meta.setdefault("fulltext_md_path", "") + meta.setdefault("error", "") + meta.setdefault("retry_count", 0) + meta.setdefault("last_error", None) + meta.setdefault("last_attempt_at", None) return meta + def validate_ocr_meta(paths: dict[str, Path], meta: dict) -> tuple[str, str]: - status = str(meta.get('ocr_status', 'pending') or 'pending').strip().lower() - if status != 'done': - return (status, str(meta.get('error', '') or '')) - key = str(meta.get('zotero_key', '') or '').strip() + status = str(meta.get("ocr_status", "pending") or "pending").strip().lower() + if status != "done": + return (status, str(meta.get("error", "") or "")) + key = str(meta.get("zotero_key", "") or "").strip() if not key: - return ('done_incomplete', 'Missing zotero_key in OCR meta') - ocr_root = paths['ocr'] / key - fulltext_path = ocr_root / 'fulltext.md' - json_path = ocr_root / 'json' / 'result.json' - page_count = int(meta.get('page_count', 0) or 0) + return ("done_incomplete", "Missing zotero_key in OCR meta") + ocr_root = paths["ocr"] / key + fulltext_path = ocr_root / "fulltext.md" + json_path = ocr_root / "json" / "result.json" + page_count = int(meta.get("page_count", 0) or 0) if not fulltext_path.exists(): - return ('done_incomplete', 'OCR fulltext.md missing') + return ("done_incomplete", "OCR fulltext.md missing") if not json_path.exists(): - return ('done_incomplete', 'OCR result.json missing') + return ("done_incomplete", "OCR result.json missing") fulltext_size = fulltext_path.stat().st_size json_size = json_path.stat().st_size if page_count < 1: - return ('done_incomplete', 'OCR page_count invalid') + return ("done_incomplete", "OCR page_count invalid") if fulltext_size < 500: - return ('done_incomplete', 'OCR fulltext.md too small') + return ("done_incomplete", "OCR fulltext.md too small") if json_size < 1000: - return ('done_incomplete', 'OCR result.json too small') + return ("done_incomplete", "OCR result.json too small") try: - rendered_pages = fulltext_path.read_text(encoding='utf-8').count(''] + caption_linked_media_ids = { + item.get("block_id") + for media_list in list(figure_caption_map.values()) + list(table_caption_map.values()) + for item in media_list + } + rendered: list[str] = [f""] reference_blocks: list[dict] = [] reference_continuations: list[dict] = [] footnotes: list[str] = [] @@ -924,26 +1105,35 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P affiliation_buffer: list[str] = [] deferred_meta: list[str] = [] for block in blocks: - label = block.get('block_label', '') - content = block.get('block_content', '') - bbox = block.get('block_bbox', [0, 0, 0, 0]) - composite_region = composite_by_block_id.get(block.get('block_id')) + label = block.get("block_label", "") + content = block.get("block_content", "") + bbox = block.get("block_bbox", [0, 0, 0, 0]) + composite_region = composite_by_block_id.get(block.get("block_id")) if composite_region: - if not composite_region.get('rendered'): - composite_region['rendered'] = True - region_bbox = composite_region['bbox'] - asset_path = images_dir / 'blocks' / f'page_{page_index:03d}_figure_{region_bbox[0]}_{region_bbox[1]}_{region_bbox[2]}_{region_bbox[3]}.jpg' + if not composite_region.get("rendered"): + composite_region["rendered"] = True + region_bbox = composite_region["bbox"] + asset_path = ( + images_dir + / "blocks" + / f"page_{page_index:03d}_figure_{region_bbox[0]}_{region_bbox[1]}_{region_bbox[2]}_{region_bbox[3]}.jpg" + ) if page_image and crop_block_asset(page_image, region_bbox, asset_path): rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault))) continue - if label in {'header', 'header_image', 'footer', 'footer_image', 'number'}: + if label in {"header", "header_image", "footer", "footer_image", "number"}: continue - if label == 'doc_title': - rendered.append(f'# {clean_block_text(content)}') + if label == "doc_title": + rendered.append(f"# {clean_block_text(content)}") continue - if label == 'paragraph_title': + if label == "paragraph_title": title = clean_block_text(content) - if title.lower() == 'check for updates' or is_subfigure_label(title) or is_reference_tail_noise_line(title) or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height): + if ( + title.lower() == "check for updates" + or is_subfigure_label(title) + or is_reference_tail_noise_line(title) + or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height) + ): continue if page_index == 1 and affiliation_buffer: rendered.extend(affiliation_buffer) @@ -952,9 +1142,9 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P rendered.extend(deferred_meta) deferred_meta.clear() first_page_meta_done = True - rendered.append(f'### {title}') + rendered.append(f"### {title}") continue - if label == 'abstract': + if label == "abstract": if page_index == 1 and affiliation_buffer: rendered.extend(affiliation_buffer) affiliation_buffer.clear() @@ -964,7 +1154,7 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P deferred_meta.clear() first_page_meta_done = True continue - if label == 'reference_content': + if label == "reference_content": text = clean_block_text(content) if text and (not is_reference_tail_noise_line(text)): if parse_reference_number(text) is None: @@ -972,9 +1162,15 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P else: reference_blocks.append(block) continue - if label == 'text': + if label == "text": text = clean_block_text(content) - if not text or is_subfigure_label(text) or is_frontmatter_noise_line(text) or is_reference_tail_noise_line(text) or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height): + if ( + not text + or is_subfigure_label(text) + or is_frontmatter_noise_line(text) + or is_reference_tail_noise_line(text) + or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height) + ): continue if raw_reference_blocks and bbox[1] >= first_reference_y - 10: reference_continuations.append(block) @@ -984,59 +1180,75 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P continue rendered.append(text) continue - if label == 'display_formula': + if label == "display_formula": formula = clean_block_text(content) formula = formula.strip() - if formula.startswith('$$') and formula.endswith('$$') and (len(formula) >= 4): + if formula.startswith("$$") and formula.endswith("$$") and (len(formula) >= 4): formula = formula[2:-2].strip() - rendered.append(f'$$\n{formula}\n$$') + rendered.append(f"$$\n{formula}\n$$") continue - if label == 'formula_number': + if label == "formula_number": rendered.append(clean_block_text(content)) continue - if label in {'table', 'image', 'chart'}: - if block.get('block_id') in caption_linked_media_ids: + if label in {"table", "image", "chart"}: + if block.get("block_id") in caption_linked_media_ids: continue - if block.get('block_id') in rendered_caption_media_ids: + if block.get("block_id") in rendered_caption_media_ids: continue - if label in {'image', 'chart'}: - cluster_id = cluster_index.get(block.get('block_id', -1)) + if label in {"image", "chart"}: + cluster_id = cluster_index.get(block.get("block_id", -1)) if cluster_id is None or cluster_id in rendered_cluster_ids: continue rendered_cluster_ids.add(cluster_id) cluster_blocks = clusters[cluster_id] - bbox = [min((item['block_bbox'][0] for item in cluster_blocks)), min((item['block_bbox'][1] for item in cluster_blocks)), max((item['block_bbox'][2] for item in cluster_blocks)), max((item['block_bbox'][3] for item in cluster_blocks))] - asset_name = f'page_{page_index:03d}_figure_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg' + bbox = [ + min(item["block_bbox"][0] for item in cluster_blocks), + min(item["block_bbox"][1] for item in cluster_blocks), + max(item["block_bbox"][2] for item in cluster_blocks), + max(item["block_bbox"][3] for item in cluster_blocks), + ] + asset_name = f"page_{page_index:03d}_figure_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg" else: - asset_name = f'page_{page_index:03d}_{label}_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg' - asset_path = images_dir / 'blocks' / asset_name + asset_name = f"page_{page_index:03d}_{label}_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg" + asset_path = images_dir / "blocks" / asset_name if page_image and crop_block_asset(page_image, bbox, asset_path): rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault))) - if label == 'table' and content: + if label == "table" and content: rendered.append(clean_block_text(content)) continue - if label == 'figure_title': + if label == "figure_title": caption_text = clean_block_text(content) if is_subfigure_label(caption_text): continue if not is_formal_figure_legend(caption_text): continue - linked_media = figure_caption_map.get(block.get('block_id'), []) or table_caption_map.get(block.get('block_id'), []) + linked_media = figure_caption_map.get(block.get("block_id"), []) or table_caption_map.get( + block.get("block_id"), [] + ) if linked_media and page_image: - rendered_caption_media_ids.update((item.get('block_id') for item in linked_media)) - union_bbox = [min((item['block_bbox'][0] for item in linked_media)), min((item['block_bbox'][1] for item in linked_media)), max((item['block_bbox'][2] for item in linked_media)), max((item['block_bbox'][3] for item in linked_media))] - asset_kind = 'figure' if block.get('block_id') in figure_caption_map else 'table' - asset_path = images_dir / 'blocks' / f'page_{page_index:03d}_{asset_kind}_{union_bbox[0]}_{union_bbox[1]}_{union_bbox[2]}_{union_bbox[3]}.jpg' + rendered_caption_media_ids.update(item.get("block_id") for item in linked_media) + union_bbox = [ + min(item["block_bbox"][0] for item in linked_media), + min(item["block_bbox"][1] for item in linked_media), + max(item["block_bbox"][2] for item in linked_media), + max(item["block_bbox"][3] for item in linked_media), + ] + asset_kind = "figure" if block.get("block_id") in figure_caption_map else "table" + asset_path = ( + images_dir + / "blocks" + / f"page_{page_index:03d}_{asset_kind}_{union_bbox[0]}_{union_bbox[1]}_{union_bbox[2]}_{union_bbox[3]}.jpg" + ) if crop_block_asset(page_image, union_bbox, asset_path): rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault))) - if asset_kind == 'table': + if asset_kind == "table": for item in linked_media: - if item.get('block_label') == 'table' and item.get('block_content'): - rendered.append(clean_block_text(item.get('block_content', ''))) + if item.get("block_label") == "table" and item.get("block_content"): + rendered.append(clean_block_text(item.get("block_content", ""))) break rendered.append(caption_text) continue - if label == 'vision_footnote': + if label == "vision_footnote": if is_formal_figure_legend(content): rendered.append(clean_block_text(content)) continue @@ -1046,30 +1258,34 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P marker_to_id = {} for marker, body in entries: footnote_counter += 1 - footnote_id = f'p{page_index}-fn{footnote_counter}' + footnote_id = f"p{page_index}-fn{footnote_counter}" marker_to_id[marker] = footnote_id - footnotes.append(f'[^{footnote_id}]: {body or clean_block_text(content)}') + footnotes.append(f"[^{footnote_id}]: {body or clean_block_text(content)}") if rendered and marker_to_id: last = rendered[-1] - if last.startswith(''): + if last.startswith("
"): rendered[-1] = attach_table_footnotes(last, marker_to_id) else: for marker, footnote_id in marker_to_id.items(): rendered[-1] = attach_footnote_reference(rendered[-1], marker, footnote_id) continue if footnotes: - rendered.append('') + rendered.append("") rendered.extend(footnotes) rendered = dedupe_page_media_lines(rendered) if reference_blocks: - rendered.append('') + rendered.append("") ordered_reference_blocks = sort_reference_blocks(reference_blocks) continuation_map: dict[int, list[str]] = {} sorted_continuations = sorted(reference_continuations, key=block_sort_key) assigned_continuation_indexes: set[int] = set() - incomplete_reference_indexes = [index for index, block in enumerate(ordered_reference_blocks) if clean_block_text(block.get('block_content', '')).rstrip().endswith(':')] - for index, continuation in zip(incomplete_reference_indexes, sorted_continuations): - continuation_text = clean_block_text(continuation.get('block_content', '')) + incomplete_reference_indexes = [ + index + for index, block in enumerate(ordered_reference_blocks) + if clean_block_text(block.get("block_content", "")).rstrip().endswith(":") + ] + for index, continuation in zip(incomplete_reference_indexes, sorted_continuations, strict=False): + continuation_text = clean_block_text(continuation.get("block_content", "")) if continuation_text: continuation_map.setdefault(index, []).append(continuation_text) assigned_continuation_indexes.add(id(continuation)) @@ -1079,270 +1295,298 @@ def render_page_blocks(vault: Path, page_index: int, result: dict, images_dir: P target_index = assign_reference_continuation(continuation, ordered_reference_blocks) if target_index is None: continue - continuation_map.setdefault(target_index, []).append(clean_block_text(continuation.get('block_content', ''))) + continuation_map.setdefault(target_index, []).append( + clean_block_text(continuation.get("block_content", "")) + ) assigned_continuation_indexes.add(id(continuation)) - unassigned_continuations = [clean_block_text(continuation.get('block_content', '')) for continuation in sorted_continuations if id(continuation) not in assigned_continuation_indexes] + unassigned_continuations = [ + clean_block_text(continuation.get("block_content", "")) + for continuation in sorted_continuations + if id(continuation) not in assigned_continuation_indexes + ] for index, block in enumerate(ordered_reference_blocks): - text = clean_block_text(block.get('block_content', '')) + text = clean_block_text(block.get("block_content", "")) if text: rendered.append(text) for continuation_text in continuation_map.get(index, []): if continuation_text: rendered.append(continuation_text) - if text.rstrip().endswith(':') and unassigned_continuations: + if text.rstrip().endswith(":") and unassigned_continuations: rendered.append(unassigned_continuations.pop(0)) return [part for part in rendered if part] + def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tuple[int, str, str, str]: paths = pipeline_paths(vault) - ocr_root = paths['ocr'] / key - json_dir = ocr_root / 'json' - images_dir = ocr_root / 'images' - page_cache_dir = ocr_root / 'pages' - meta_path = ocr_root / 'meta.json' + ocr_root = paths["ocr"] / key + json_dir = ocr_root / "json" + images_dir = ocr_root / "images" + page_cache_dir = ocr_root / "pages" + meta_path = ocr_root / "meta.json" json_dir.mkdir(parents=True, exist_ok=True) images_dir.mkdir(parents=True, exist_ok=True) page_cache_dir.mkdir(parents=True, exist_ok=True) page_num = 0 merged_parts = [] meta = read_json(meta_path) if meta_path.exists() else {} - source_pdf = Path(meta.get('source_pdf', '')) if meta.get('source_pdf') else None + source_pdf = Path(meta.get("source_pdf", "")) if meta.get("source_pdf") else None pdf_doc = None try: if source_pdf and source_pdf.exists(): pdf_doc = fitz.open(str(source_pdf)) for page_payload in all_results: - for res in page_payload.get('layoutParsingResults', []): + for res in page_payload.get("layoutParsingResults", []): page_num += 1 - merged_parts.append('\n\n'.join(render_page_blocks(vault, page_num, res, images_dir, page_cache_dir, pdf_doc=pdf_doc))) + merged_parts.append( + "\n\n".join(render_page_blocks(vault, page_num, res, images_dir, page_cache_dir, pdf_doc=pdf_doc)) + ) finally: if pdf_doc is not None: pdf_doc.close() - write_json(json_dir / 'result.json', all_results) - fulltext_path = ocr_root / 'fulltext.md' - fulltext_path.write_text('\n\n'.join(merged_parts).strip() + '\n', encoding='utf-8') - markdown_dir = ocr_root / 'markdown' + write_json(json_dir / "result.json", all_results) + fulltext_path = ocr_root / "fulltext.md" + fulltext_path.write_text("\n\n".join(merged_parts).strip() + "\n", encoding="utf-8") + markdown_dir = ocr_root / "markdown" if markdown_dir.exists(): shutil.rmtree(markdown_dir) - markdown_path = str(fulltext_path.relative_to(vault)).replace('\\', '/') if page_num else '' - json_path = str((json_dir / 'result.json').relative_to(vault)).replace('\\', '/') + markdown_path = str(fulltext_path.relative_to(vault)).replace("\\", "/") if page_num else "" + json_path = str((json_dir / "result.json").relative_to(vault)).replace("\\", "/") fulltext_md_path = str(fulltext_path.resolve()) return (page_num, markdown_path, json_path, fulltext_md_path) + def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False) -> int: from paperforge.pdf_resolver import resolve_pdf_path + paths = pipeline_paths(vault) cleanup_blocked_ocr_dirs(paths) # Zombie reset: reset stale processing jobs to pending - zombie_timeout = int(os.environ.get('PAPERFORGE_ZOMBIE_TIMEOUT_MINUTES', '30')) - ocr_root = paths.get('ocr') + zombie_timeout = int(os.environ.get("PAPERFORGE_ZOMBIE_TIMEOUT_MINUTES", "30")) + ocr_root = paths.get("ocr") if ocr_root and ocr_root.exists(): for meta_dir in ocr_root.iterdir(): - meta_path = meta_dir / 'meta.json' + meta_path = meta_dir / "meta.json" if not meta_path.exists(): continue try: meta = read_json(meta_path) except Exception: continue - zkey = meta.get('zotero_key', meta_dir.name) - zstatus = str(meta.get('ocr_status', '') or '').strip().lower() - if zstatus in {'queued', 'running'}: - zstarted = meta.get('ocr_started_at', '') + zkey = meta.get("zotero_key", meta_dir.name) + zstatus = str(meta.get("ocr_status", "") or "").strip().lower() + if zstatus in {"queued", "running"}: + zstarted = meta.get("ocr_started_at", "") if zstarted: try: started_dt = datetime.fromisoformat(zstarted) if (datetime.now(timezone.utc) - started_dt).total_seconds() > zombie_timeout * 60: - meta['ocr_status'] = 'pending' - meta['retry_count'] = int(meta.get('retry_count', 0)) + 1 + meta["ocr_status"] = "pending" + meta["retry_count"] = int(meta.get("retry_count", 0)) + 1 write_json(meta_path, meta) - logger.warning("Zombie reset %s: was %s, started at %s, reset to pending", zkey, zstatus, zstarted) + logger.warning( + "Zombie reset %s: was %s, started at %s, reset to pending", zkey, zstatus, zstarted + ) except Exception: pass control_actions = load_control_actions(paths) - target_keys = {key for key, action in control_actions.items() if action.get('do_ocr', False)} + target_keys = {key for key, action in control_actions.items() if action.get("do_ocr", False)} target_rows = [] - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): for item in load_export_rows(export_path): - if item['key'] not in target_keys: + if item["key"] not in target_keys: continue - pdf_attachments = [a for a in item.get('attachments', []) if a.get('contentType') == 'application/pdf'] - target_rows.append({'zotero_key': item['key'], 'has_pdf': bool(pdf_attachments), 'pdf_path': pdf_attachments[0]['path'] if pdf_attachments else ''}) + pdf_attachments = [a for a in item.get("attachments", []) if a.get("contentType") == "application/pdf"] + target_rows.append( + { + "zotero_key": item["key"], + "has_pdf": bool(pdf_attachments), + "pdf_path": pdf_attachments[0]["path"] if pdf_attachments else "", + } + ) ocr_queue = sync_ocr_queue(paths, target_rows) - max_items_raw = os.environ.get('PADDLEOCR_MAX_ITEMS', '').strip() + max_items_raw = os.environ.get("PADDLEOCR_MAX_ITEMS", "").strip() max_items = 3 if max_items_raw: try: max_items = max(1, int(max_items_raw)) except ValueError: max_items = 3 - token = os.environ.get('PADDLEOCR_API_TOKEN', '').strip() + token = os.environ.get("PADDLEOCR_API_TOKEN", "").strip() if not token: - token = os.environ.get('PADDLEOCR_API_TOKEN_USER', '').strip() + token = os.environ.get("PADDLEOCR_API_TOKEN_USER", "").strip() if not token: try: import winreg - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, 'Environment') as env_key: - token = str(winreg.QueryValueEx(env_key, 'PADDLEOCR_API_TOKEN')[0]).strip() + + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Environment") as env_key: + token = str(winreg.QueryValueEx(env_key, "PADDLEOCR_API_TOKEN")[0]).strip() except Exception: - token = '' - job_url = os.environ.get('PADDLEOCR_JOB_URL', 'https://paddleocr.aistudio-app.com/api/v2/ocr/jobs').strip() - model = os.environ.get('PADDLEOCR_MODEL', 'PaddleOCR-VL-1.5').strip() - optional_payload = {'useDocOrientationClassify': False, 'useDocUnwarping': False, 'useChartRecognition': False} + token = "" + job_url = os.environ.get("PADDLEOCR_JOB_URL", "https://paddleocr.aistudio-app.com/api/v2/ocr/jobs").strip() + model = os.environ.get("PADDLEOCR_MODEL", "PaddleOCR-VL-1.5").strip() + optional_payload = {"useDocOrientationClassify": False, "useDocUnwarping": False, "useChartRecognition": False} changed = 0 active_submitted = 0 queue_changed = False def _do_poll(job_id: str, token_val: str) -> requests.Response: - resp = requests.get(f"{job_url}/{job_id}", headers={'Authorization': f'bearer {token_val}'}, timeout=60) + resp = requests.get(f"{job_url}/{job_id}", headers={"Authorization": f"bearer {token_val}"}, timeout=60) resp.raise_for_status() return resp for queue_row in progress_bar(ocr_queue, desc="Processing OCR", disable=no_progress): - key = queue_row['zotero_key'] + key = queue_row["zotero_key"] meta = ensure_ocr_meta(vault, queue_row) - status = str(meta.get('ocr_status', 'pending') or 'pending').strip().lower() - queue_row['queue_status'] = status - if status == 'done': + status = str(meta.get("ocr_status", "pending") or "pending").strip().lower() + queue_row["queue_status"] = status + if status == "done": queue_changed = True continue - if status in {'queued', 'running'} and meta.get('ocr_job_id'): + if status in {"queued", "running"} and meta.get("ocr_job_id"): active_submitted += 1 if not token: continue - meta_path_poll = paths['ocr'] / key / 'meta.json' + meta_path_poll = paths["ocr"] / key / "meta.json" try: - response = retry_with_meta(_do_poll, meta_path_poll, meta['ocr_job_id'], token) + response = retry_with_meta(_do_poll, meta_path_poll, meta["ocr_job_id"], token) except Exception as e: from paperforge.ocr_diagnostics import classify_error - state, suggestion = classify_error(e, getattr(e, 'response', None)) - meta['ocr_status'] = state - meta['error'] = str(e) - meta['suggestion'] = suggestion - meta['library_record'] = key - queue_row['queue_status'] = state - write_json(paths['ocr'] / key / 'meta.json', meta) + + state, suggestion = classify_error(e, getattr(e, "response", None)) + meta["ocr_status"] = state + meta["error"] = str(e) + meta["suggestion"] = suggestion + meta["library_record"] = key + queue_row["queue_status"] = state + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 active_submitted = max(0, active_submitted - 1) continue try: - payload = response.json()['data'] - state = payload['state'] + payload = response.json()["data"] + state = payload["state"] except (json.JSONDecodeError, KeyError) as e: from paperforge.ocr_diagnostics import classify_error + state, suggestion = classify_error(e, None) - meta['ocr_status'] = state - meta['error'] = f'API schema mismatch during polling: {e}' - meta['suggestion'] = suggestion - meta['library_record'] = key - meta['raw_response'] = response.text[:1000] - queue_row['queue_status'] = state - write_json(paths['ocr'] / key / 'meta.json', meta) + meta["ocr_status"] = state + meta["error"] = f"API schema mismatch during polling: {e}" + meta["suggestion"] = suggestion + meta["library_record"] = key + meta["raw_response"] = response.text[:1000] + queue_row["queue_status"] = state + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 active_submitted = max(0, active_submitted - 1) continue - if state in {'pending', 'running'}: - meta['ocr_status'] = state - queue_row['queue_status'] = state - meta['error'] = '' - elif state == 'done': - result_url = payload['resultUrl']['jsonUrl'] + if state in {"pending", "running"}: + meta["ocr_status"] = state + queue_row["queue_status"] = state + meta["error"] = "" + elif state == "done": + result_url = payload["resultUrl"]["jsonUrl"] result_response = requests.get(result_url, timeout=120) result_response.raise_for_status() lines = [line.strip() for line in result_response.text.splitlines() if line.strip()] all_results = [] for line in lines: - page_payload = json.loads(line)['result'] + page_payload = json.loads(line)["result"] all_results.append(page_payload) page_num, markdown_path, json_path, fulltext_md_path = postprocess_ocr_result(vault, key, all_results) - meta['ocr_status'] = 'done' - meta['ocr_finished_at'] = datetime.now(timezone.utc).isoformat() - meta['page_count'] = page_num - meta['markdown_path'] = markdown_path - meta['json_path'] = json_path - meta['fulltext_md_path'] = fulltext_md_path - meta['error'] = '' - queue_row['queue_status'] = 'done' + meta["ocr_status"] = "done" + meta["ocr_finished_at"] = datetime.now(timezone.utc).isoformat() + meta["page_count"] = page_num + meta["markdown_path"] = markdown_path + meta["json_path"] = json_path + meta["fulltext_md_path"] = fulltext_md_path + meta["error"] = "" + queue_row["queue_status"] = "done" queue_changed = True active_submitted = max(0, active_submitted - 1) else: - meta['ocr_status'] = 'error' - meta['error'] = payload.get('errorMsg', 'Unknown OCR failure') - meta['library_record'] = key - queue_row['queue_status'] = 'error' + meta["ocr_status"] = "error" + meta["error"] = payload.get("errorMsg", "Unknown OCR failure") + meta["library_record"] = key + queue_row["queue_status"] = "error" active_submitted = max(0, active_submitted - 1) - write_json(paths['ocr'] / key / 'meta.json', meta) + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 available_slots = max(0, max_items - active_submitted) if available_slots > 0: def _do_upload(token_val: str, pdf_path: Path) -> requests.Response: - with open(pdf_path, 'rb') as file_handle: - resp = requests.post(job_url, headers={'Authorization': f'bearer {token_val}'}, data={'model': model, 'optionalPayload': json.dumps(optional_payload)}, files={'file': file_handle}, timeout=120) + with open(pdf_path, "rb") as file_handle: + resp = requests.post( + job_url, + headers={"Authorization": f"bearer {token_val}"}, + data={"model": model, "optionalPayload": json.dumps(optional_payload)}, + files={"file": file_handle}, + timeout=120, + ) resp.raise_for_status() return resp - upload_items = [r for r in ocr_queue if r.get('queue_status', '') not in ('done', 'queued', 'running')] + upload_items = [r for r in ocr_queue if r.get("queue_status", "") not in ("done", "queued", "running")] for queue_row in progress_bar(upload_items, desc="Uploading", disable=no_progress): if available_slots <= 0: break - key = queue_row['zotero_key'] + key = queue_row["zotero_key"] meta = ensure_ocr_meta(vault, queue_row) - status = str(meta.get('ocr_status', 'pending') or 'pending').strip().lower() - if status == 'done': + status = str(meta.get("ocr_status", "pending") or "pending").strip().lower() + if status == "done": queue_changed = True continue - if status in {'queued', 'running'} and meta.get('ocr_job_id'): + if status in {"queued", "running"} and meta.get("ocr_job_id"): continue resolved_pdf = resolve_pdf_path( - queue_row.get('pdf_path', ''), - queue_row.get('has_pdf', False), + queue_row.get("pdf_path", ""), + queue_row.get("has_pdf", False), vault, - paths.get('zotero_dir') if 'zotero_dir' in paths else None, + paths.get("zotero_dir") if "zotero_dir" in paths else None, ) if not resolved_pdf: - meta['ocr_status'] = 'nopdf' - meta['error'] = 'PDF not found or not readable' - queue_row['queue_status'] = 'nopdf' - write_json(paths['ocr'] / key / 'meta.json', meta) + meta["ocr_status"] = "nopdf" + meta["error"] = "PDF not found or not readable" + queue_row["queue_status"] = "nopdf" + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 continue if not token: - meta['ocr_status'] = 'blocked' - meta['error'] = 'PaddleOCR not configured' - queue_row['queue_status'] = 'blocked' - write_json(paths['ocr'] / key / 'meta.json', meta) + meta["ocr_status"] = "blocked" + meta["error"] = "PaddleOCR not configured" + queue_row["queue_status"] = "blocked" + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 continue - meta_path_upload = paths['ocr'] / key / 'meta.json' + meta_path_upload = paths["ocr"] / key / "meta.json" try: response = retry_with_meta(_do_upload, meta_path_upload, token, resolved_pdf) except Exception as e: from paperforge.ocr_diagnostics import classify_error - state, suggestion = classify_error(e, getattr(e, 'response', None)) - meta['ocr_status'] = state - meta['error'] = str(e) - meta['suggestion'] = suggestion - meta['library_record'] = key - queue_row['queue_status'] = state - write_json(paths['ocr'] / key / 'meta.json', meta) + + state, suggestion = classify_error(e, getattr(e, "response", None)) + meta["ocr_status"] = state + meta["error"] = str(e) + meta["suggestion"] = suggestion + meta["library_record"] = key + queue_row["queue_status"] = state + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 continue - meta['ocr_job_id'] = response.json()['data']['jobId'] - meta['ocr_status'] = 'queued' - meta['ocr_started_at'] = datetime.now(timezone.utc).isoformat() - meta['error'] = '' - queue_row['queue_status'] = 'queued' - write_json(paths['ocr'] / key / 'meta.json', meta) + meta["ocr_job_id"] = response.json()["data"]["jobId"] + meta["ocr_status"] = "queued" + meta["ocr_started_at"] = datetime.now(timezone.utc).isoformat() + meta["error"] = "" + queue_row["queue_status"] = "queued" + write_json(paths["ocr"] / key / "meta.json", meta) changed += 1 available_slots -= 1 if queue_changed: - ocr_queue = [row for row in ocr_queue if str(row.get('queue_status', '')).lower() != 'done'] + ocr_queue = [row for row in ocr_queue if str(row.get("queue_status", "")).lower() != "done"] write_ocr_queue(paths, ocr_queue) _sync.run_selection_sync(vault) _sync.run_index_refresh(vault) - print(f'ocr: updated {changed} records') + print(f"ocr: updated {changed} records") return 0 - diff --git a/paperforge/worker/repair.py b/paperforge/worker/repair.py index cad41099..6d192d06 100644 --- a/paperforge/worker/repair.py +++ b/paperforge/worker/repair.py @@ -1,155 +1,24 @@ from __future__ import annotations -import logging -import argparse -import csv -import hashlib -import html -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import urllib.parse -from json import JSONDecodeError -import zipfile -from datetime import datetime, timezone -from pathlib import Path -from xml.etree import ElementTree as ET -import requests -import fitz -from PIL import Image +import logging +import re +from pathlib import Path + +from paperforge.config import load_vault_config, paperforge_paths +from paperforge.worker._utils import ( + _resolve_formal_note_path, + read_json, + write_json, +) +from paperforge.worker.ocr import validate_ocr_meta from paperforge.worker.sync import ( load_export_rows, obsidian_wikilink_for_pdf, update_frontmatter_field, ) -from paperforge.worker.deep_reading import _resolve_formal_note_path -from paperforge.worker.ocr import validate_ocr_meta logger = logging.getLogger(__name__) -STANDARD_VIEW_NAMES = frozenset([ - "控制面板", "推荐分析", "待 OCR", "OCR 完成", - "待深度阅读", "深度阅读完成", "正式卡片", "全记录" -]) - -def load_simple_env(env_path: Path) -> None: - if not env_path.exists(): - return - for raw_line in env_path.read_text(encoding='utf-8').splitlines(): - line = raw_line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - key, value = line.split('=', 1) - key = key.strip() - if not key or key in os.environ: - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}): - value = value[1:-1] - os.environ[key] = value - -def read_json(path: Path): - return json.loads(path.read_text(encoding='utf-8')) -_JOURNAL_DB: dict[str, dict] | None = None - -def load_journal_db(vault: Path) -> dict[str, dict]: - """Load zoterostyle.json journal database.""" - global _JOURNAL_DB - if _JOURNAL_DB is not None: - return _JOURNAL_DB - zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json' - if zoterostyle_path.exists(): - try: - _JOURNAL_DB = read_json(zoterostyle_path) - except (JSONDecodeError, Exception): - _JOURNAL_DB = {} - else: - _JOURNAL_DB = {} - return _JOURNAL_DB - -def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str: - """Lookup impact factor: prefer zoterostyle.json, fallback to extra field.""" - if not journal_name: - return '' - journal_db = load_journal_db(vault) - if journal_name in journal_db: - rank_data = journal_db[journal_name].get('rank', {}) - if isinstance(rank_data, dict): - sciif = rank_data.get('sciif', '') - if sciif: - return str(sciif) - if extra: - if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra) - if if_match: - return if_match.group(1) - return '' - -def write_json(path: Path, data) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') - -def read_jsonl(path: Path): - rows = [] - if not path.exists(): - return rows - for line in path.read_text(encoding='utf-8').splitlines(): - line = line.strip() - if line: - rows.append(json.loads(line)) - return rows - -def write_jsonl(path: Path, rows) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows)) - if text: - text += '\n' - path.write_text(text, encoding='utf-8') - -def yaml_quote(value: str) -> str: - if isinstance(value, bool): - return 'true' if value else 'false' - return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"' - -def yaml_block(value: str) -> list[str]: - value = (value or '').strip() - if not value: - return ['abstract: |-', ' '] - lines = ['abstract: |-'] - for line in value.splitlines(): - lines.append(f' {line}') - return lines - -def yaml_list(key: str, values) -> list[str]: - cleaned = [str(value).strip() for value in values or [] if str(value).strip()] - if not cleaned: - return [f'{key}: []'] - lines = [f'{key}:'] - for value in cleaned: - lines.append(f' - {yaml_quote(value)}') - return lines - -def slugify_filename(text: str) -> str: - cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip() - return cleaned[:120] or 'untitled' - -def _extract_year(value: str) -> str: - match = re.search('(19|20)\\d{2}', value or '') - return match.group(0) if match else '' - - -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -157,14 +26,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -191,17 +53,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -212,6 +72,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + def _find_export_for_domain(paths: dict[str, Path], domain: str) -> Path | None: """Find the BBT export JSON file for a given domain.""" for export_path in sorted(paths["exports"].glob("*.json")): @@ -313,16 +174,10 @@ def repair_pdf_paths( if item: pdf_path = item.get("pdf_path", "") if pdf_path: - new_wikilink = obsidian_wikilink_for_pdf( - pdf_path, vault, zotero_dir - ) + new_wikilink = obsidian_wikilink_for_pdf(pdf_path, vault, zotero_dir) if new_wikilink: - new_text = update_frontmatter_field( - text, "pdf_path", new_wikilink - ) - new_text = update_frontmatter_field( - new_text, "path_error", "" - ) + new_text = update_frontmatter_field(text, "pdf_path", new_wikilink) + new_text = update_frontmatter_field(new_text, "path_error", "") new_text = update_frontmatter_field( new_text, "bbt_path_raw", @@ -380,33 +235,32 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals """ result = {"scanned": 0, "divergent": [], "fixed": 0, "errors": []} config = load_domain_config(paths) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} - record_paths = list(paths['library_records'].rglob('*.md')) + {entry["export_file"]: entry["domain"] for entry in config["domains"]} + record_paths = list(paths["library_records"].rglob("*.md")) for record_path in record_paths: try: - record_text = record_path.read_text(encoding='utf-8') + record_text = record_path.read_text(encoding="utf-8") except Exception as e: - result['errors'].append({"file": str(record_path), "error": str(e)}) + result["errors"].append({"file": str(record_path), "error": str(e)}) continue key_match = re.search('^zotero_key:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE) if not key_match: continue zotero_key = key_match.group(1).strip() domain = record_path.parent.name - record_dir = record_path.parent - result['scanned'] += 1 + result["scanned"] += 1 lib_ocr_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE) - lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else 'pending' + lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else "pending" note_path = _resolve_formal_note_path(vault, zotero_key, domain) note_ocr_status = None if note_path and note_path.exists(): try: - note_text = note_path.read_text(encoding='utf-8') + note_text = note_path.read_text(encoding="utf-8") note_status_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', note_text, re.MULTILINE) note_ocr_status = note_status_match.group(1).strip() if note_status_match else None except Exception: pass - meta_path = paths['ocr'] / zotero_key / 'meta.json' + meta_path = paths["ocr"] / zotero_key / "meta.json" meta_ocr_status = None meta_validated_status = None validated_status = None @@ -418,25 +272,30 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals meta_validated_status = validated_status if validated_error and verbose: logger.warning("%s meta validation error: %s", zotero_key, validated_error) - raw_status = str(meta.get('ocr_status', '') or '').strip().lower() + raw_status = str(meta.get("ocr_status", "") or "").strip().lower() meta_ocr_status = raw_status if raw_status else None - if meta_validated_status == 'done_incomplete': - meta_ocr_status = 'done_incomplete' + if meta_validated_status == "done_incomplete": + meta_ocr_status = "done_incomplete" except Exception as e: - result['errors'].append({"file": str(meta_path), "error": str(e)}) + result["errors"].append({"file": str(meta_path), "error": str(e)}) meta_ocr_status = None is_divergent = False div_reason = "" - if meta_validated_status == 'done_incomplete': + if meta_validated_status == "done_incomplete": is_divergent = True div_reason = f"meta validation: done_incomplete ({validated_error})" - elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing', None): + elif lib_ocr_status == "done" and meta_ocr_status in ("pending", "processing", None): is_divergent = True div_reason = f"library_record done but meta {meta_ocr_status or 'missing'}" - elif note_ocr_status == 'done' and (meta_ocr_status is None or meta_validated_status == 'done_incomplete'): + elif note_ocr_status == "done" and (meta_ocr_status is None or meta_validated_status == "done_incomplete"): is_divergent = True div_reason = "formal_note done but meta.json missing/invalid" - elif lib_ocr_status != 'pending' and meta_ocr_status is not None and meta_validated_status is not None and lib_ocr_status != meta_validated_status: + elif ( + lib_ocr_status != "pending" + and meta_ocr_status is not None + and meta_validated_status is not None + and lib_ocr_status != meta_validated_status + ): is_divergent = True div_reason = f"library_record={lib_ocr_status} vs meta post-validation={meta_validated_status}" if is_divergent: @@ -448,92 +307,86 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals "meta_ocr_status": meta_validated_status or meta_ocr_status, "reason": div_reason, } - result['divergent'].append(item) + result["divergent"].append(item) if verbose: logger.info("divergent: %s | %s", zotero_key, div_reason) if fix: fixed_library_record = False fixed_formal_note = False fixed_meta = False - new_status = 'pending' - if meta_ocr_status is None or meta_validated_status == 'done_incomplete': - new_status = 'pending' - new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status) + new_status = "pending" + if meta_ocr_status is None or meta_validated_status == "done_incomplete": + new_status = "pending" + new_record_text = update_frontmatter_field(record_text, "ocr_status", new_status) if new_record_text != record_text: - record_path.write_text(new_record_text, encoding='utf-8') + record_path.write_text(new_record_text, encoding="utf-8") fixed_library_record = True if note_path and note_path.exists(): try: - note_text = note_path.read_text(encoding='utf-8') - new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status) + note_text = note_path.read_text(encoding="utf-8") + new_note_text = update_frontmatter_field(note_text, "ocr_status", new_status) if new_note_text != note_text: - note_path.write_text(new_note_text, encoding='utf-8') + note_path.write_text(new_note_text, encoding="utf-8") fixed_formal_note = True except Exception: pass - if meta_validated_status is not None and meta_validated_status != 'done': + if meta_validated_status is not None and meta_validated_status != "done": if meta_path.exists(): try: meta = read_json(meta_path) - meta['ocr_status'] = 'pending' + meta["ocr_status"] = "pending" write_json(meta_path, meta) fixed_meta = True except Exception: pass - record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE) - is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true' + record_do_ocr_match = re.search(r"^do_ocr:\s*(true|false)$", new_record_text, re.MULTILINE) + is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == "true" if not is_do_ocr: - final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true') + final_record_text = update_frontmatter_field(new_record_text, "do_ocr", "true") if final_record_text != new_record_text: - record_path.write_text(final_record_text, encoding='utf-8') + record_path.write_text(final_record_text, encoding="utf-8") fixed_library_record = True - elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing'): - new_status = 'pending' - new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status) + elif lib_ocr_status == "done" and meta_ocr_status in ("pending", "processing"): + new_status = "pending" + new_record_text = update_frontmatter_field(record_text, "ocr_status", new_status) if new_record_text != record_text: - record_path.write_text(new_record_text, encoding='utf-8') + record_path.write_text(new_record_text, encoding="utf-8") fixed_library_record = True if note_path and note_path.exists(): try: - note_text = note_path.read_text(encoding='utf-8') - new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status) + note_text = note_path.read_text(encoding="utf-8") + new_note_text = update_frontmatter_field(note_text, "ocr_status", new_status) if new_note_text != note_text: - note_path.write_text(new_note_text, encoding='utf-8') + note_path.write_text(new_note_text, encoding="utf-8") fixed_formal_note = True except Exception: pass if meta_path.exists(): try: meta = read_json(meta_path) - meta['ocr_status'] = 'pending' + meta["ocr_status"] = "pending" write_json(meta_path, meta) fixed_meta = True except Exception: pass - record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE) - is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true' + record_do_ocr_match = re.search(r"^do_ocr:\s*(true|false)$", new_record_text, re.MULTILINE) + is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == "true" if not is_do_ocr: - final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true') + final_record_text = update_frontmatter_field(new_record_text, "do_ocr", "true") if final_record_text != new_record_text: - record_path.write_text(final_record_text, encoding='utf-8') + record_path.write_text(final_record_text, encoding="utf-8") fixed_library_record = True fixed_count = sum([fixed_library_record, fixed_formal_note, fixed_meta]) - result['fixed'] += fixed_count + result["fixed"] += fixed_count if verbose and fixed_count > 0: logger.info("fixed %d files for %s", fixed_count, zotero_key) # Path error detection and repair path_errors = _detect_path_errors(paths, verbose) if path_errors["total"] > 0: - error_summary = ", ".join( - f"{count} {err}" for err, count in sorted(path_errors["by_type"].items()) - ) - print( - f"[repair] Found {path_errors['total']} items with path errors: {error_summary}" - ) + error_summary = ", ".join(f"{count} {err}" for err, count in sorted(path_errors["by_type"].items())) + print(f"[repair] Found {path_errors['total']} items with path errors: {error_summary}") if fix_paths: - fixed_count = repair_pdf_paths( - vault, paths, path_errors["records"], verbose - ) + fixed_count = repair_pdf_paths(vault, paths, path_errors["records"], verbose) print(f"[repair] Fixed {fixed_count} PDF paths") else: print("[repair] Tip: run with --fix-paths to attempt auto-resolution") @@ -542,4 +395,3 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals result["path_errors"] = path_errors return result - diff --git a/paperforge/worker/status.py b/paperforge/worker/status.py index fce4a244..ddff53b4 100644 --- a/paperforge/worker/status.py +++ b/paperforge/worker/status.py @@ -1,149 +1,22 @@ from __future__ import annotations -import logging -import argparse -import csv -import hashlib -import html + import json +import logging import os import re -import shutil -import subprocess import sys -import tempfile -import urllib.parse from json import JSONDecodeError -import zipfile -from datetime import datetime, timezone from pathlib import Path -from xml.etree import ElementTree as ET -import requests -import fitz -from PIL import Image +from paperforge.config import load_vault_config, paperforge_paths +from paperforge.worker._utils import ( + read_json, + write_json, +) from paperforge.worker.base_views import ensure_base_views logger = logging.getLogger(__name__) -STANDARD_VIEW_NAMES = frozenset([ - "控制面板", "推荐分析", "待 OCR", "OCR 完成", - "待深度阅读", "深度阅读完成", "正式卡片", "全记录" -]) - -def load_simple_env(env_path: Path) -> None: - if not env_path.exists(): - return - for raw_line in env_path.read_text(encoding='utf-8').splitlines(): - line = raw_line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - key, value = line.split('=', 1) - key = key.strip() - if not key or key in os.environ: - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}): - value = value[1:-1] - os.environ[key] = value - -def read_json(path: Path): - return json.loads(path.read_text(encoding='utf-8')) -_JOURNAL_DB: dict[str, dict] | None = None - -def load_journal_db(vault: Path) -> dict[str, dict]: - """Load zoterostyle.json journal database.""" - global _JOURNAL_DB - if _JOURNAL_DB is not None: - return _JOURNAL_DB - zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json' - if zoterostyle_path.exists(): - try: - _JOURNAL_DB = read_json(zoterostyle_path) - except (JSONDecodeError, Exception): - _JOURNAL_DB = {} - else: - _JOURNAL_DB = {} - return _JOURNAL_DB - -def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str: - """Lookup impact factor: prefer zoterostyle.json, fallback to extra field.""" - if not journal_name: - return '' - journal_db = load_journal_db(vault) - if journal_name in journal_db: - rank_data = journal_db[journal_name].get('rank', {}) - if isinstance(rank_data, dict): - sciif = rank_data.get('sciif', '') - if sciif: - return str(sciif) - if extra: - if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra) - if if_match: - return if_match.group(1) - return '' - -def write_json(path: Path, data) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') - -def read_jsonl(path: Path): - rows = [] - if not path.exists(): - return rows - for line in path.read_text(encoding='utf-8').splitlines(): - line = line.strip() - if line: - rows.append(json.loads(line)) - return rows - -def write_jsonl(path: Path, rows) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows)) - if text: - text += '\n' - path.write_text(text, encoding='utf-8') - -def yaml_quote(value: str) -> str: - if isinstance(value, bool): - return 'true' if value else 'false' - return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"' - -def yaml_block(value: str) -> list[str]: - value = (value or '').strip() - if not value: - return ['abstract: |-', ' '] - lines = ['abstract: |-'] - for line in value.splitlines(): - lines.append(f' {line}') - return lines - -def yaml_list(key: str, values) -> list[str]: - cleaned = [str(value).strip() for value in values or [] if str(value).strip()] - if not cleaned: - return [f'{key}: []'] - lines = [f'{key}:'] - for value in cleaned: - lines.append(f' - {yaml_quote(value)}') - return lines - -def slugify_filename(text: str) -> str: - cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip() - return cleaned[:120] or 'untitled' - -def _extract_year(value: str) -> str: - match = re.search('(19|20)\\d{2}', value or '') - return match.group(0) if match else '' - - -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -151,14 +24,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -185,17 +51,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -206,6 +70,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + def _detect_zotero_data_dir() -> str | None: """Try to detect the user's Zotero data directory.""" if os.name == "nt": @@ -252,7 +117,7 @@ def check_zotero_location(vault: Path, cfg: dict, add_check) -> None: "Path Resolution", "warn", f"Zotero junction target missing: {target}", - f"Recreate junction: mklink /J \"{zotero_link}\" \"\"", + f'Recreate junction: mklink /J "{zotero_link}" ""', ) except Exception: add_check( @@ -308,9 +173,7 @@ def check_pdf_paths(vault: Path, paths: dict, add_check) -> None: return import random - sample = ( - record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5) - ) + sample = record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5) valid = 0 errors: list[str] = [] for record_path in sample: @@ -329,9 +192,7 @@ def check_pdf_paths(vault: Path, paths: dict, add_check) -> None: if candidate.exists(): valid += 1 else: - err_match = re.search( - r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE - ) + err_match = re.search(r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE) error_type = err_match.group(1) if err_match else "not_found" errors.append(error_type) total = len(sample) @@ -395,7 +256,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: if sys_version.major >= 3 and sys_version.minor >= 10: add_check("Python 环境", "pass", f"Python {sys_version.major}.{sys_version.minor}.{sys_version.micro}") else: - add_check("Python 环境", "fail", f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)", "升级 Python 到 3.10 或更高版本") + add_check( + "Python 环境", + "fail", + f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)", + "升级 Python 到 3.10 或更高版本", + ) required_modules = ["requests", "pymupdf", "PIL", "yaml"] missing_modules = [] @@ -405,7 +271,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: except ImportError: missing_modules.append(mod) if missing_modules: - add_check("Python 环境", "fail", f"缺少模块: {', '.join(missing_modules)}", f"运行: pip install {' '.join(missing_modules)}") + add_check( + "Python 环境", + "fail", + f"缺少模块: {', '.join(missing_modules)}", + f"运行: pip install {' '.join(missing_modules)}", + ) elif sys_version.major >= 3 and sys_version.minor >= 10: pass @@ -424,13 +295,20 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: if resources_dir.exists(): add_check("Vault 结构", "pass", f"resources_dir 存在: {cfg['resources_dir']}") else: - add_check("Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}") + add_check( + "Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}" + ) control_dir = resources_dir / cfg.get("control_dir", "LiteratureControl") if control_dir.exists(): add_check("Vault 结构", "pass", f"control_dir 存在: {cfg.get('control_dir', 'LiteratureControl')}") else: - add_check("Vault 结构", "fail", f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}", f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}") + add_check( + "Vault 结构", + "fail", + f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}", + f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}", + ) zotero_link = system_dir / "Zotero" if zotero_link.exists(): @@ -439,14 +317,16 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: else: add_check("Zotero 链接", "warn", "Zotero 目录存在但不是 junction,建议使用 junction") else: - add_check("Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} ") + add_check( + "Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} " + ) exports_dir = system_dir / "PaperForge" / "exports" if exports_dir.exists(): add_check("BBT 导出", "pass", "exports 目录存在") else: add_check("BBT 导出", "fail", "exports 目录不存在", "在 Better BibTeX 设置中配置导出路径") - + # Check for any valid JSON export (per-domain or library.json) json_files = sorted(exports_dir.glob("*.json")) if exports_dir.exists() else [] valid_exports = [] @@ -463,7 +343,7 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: valid_exports.append((jf.name, len(data["items"]))) except (JSONDecodeError, Exception): pass - + if valid_exports: for name, count in valid_exports: add_check("BBT 导出", "pass", f"{name} 正常 ({count} 条)") @@ -472,7 +352,9 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: else: add_check("BBT 导出", "fail", "未找到 JSON 导出文件", "在 Zotero Better BibTeX 设置中配置导出路径") - env_api_key = os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN") + env_api_key = ( + os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN") + ) if env_api_key: add_check("OCR 配置", "pass", "API Token 已配置") else: @@ -480,10 +362,11 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: # Worker module check (v1.3+: pipeline/ removed, use paperforge.worker package) try: - from paperforge.worker.sync import run_selection_sync, run_index_refresh + from paperforge.worker.base_views import ensure_base_views from paperforge.worker.deep_reading import run_deep_reading from paperforge.worker.ocr import run_ocr - from paperforge.worker.base_views import ensure_base_views + from paperforge.worker.sync import run_index_refresh, run_selection_sync + add_check("Worker 脚本", "pass", "paperforge.worker 包可导入") except ImportError as e: add_check("Worker 脚本", "fail", f"worker 函数导入失败: {e}", "运行: pip install -e .") @@ -504,6 +387,7 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: if ld_deep_script and ld_deep_script.exists(): try: import importlib.util + spec = importlib.util.spec_from_file_location("ld_deep", ld_deep_script) if spec and spec.loader: mod = importlib.util.module_from_spec(spec) @@ -514,7 +398,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int: if ld_deep_import_ok: add_check("Agent 脚本", "pass", "paperforge and ld_deep importable") else: - add_check("Agent 脚本", "warn", f"literature-qa skill 目录存在但 import 失败: {import_error}", "确认 agent_config_dir 配置正确并已运行 pip install -e .") + add_check( + "Agent 脚本", + "warn", + f"literature-qa skill 目录存在但 import 失败: {import_error}", + "确认 agent_config_dir 配置正确并已运行 pip install -e .", + ) else: add_check("Agent 脚本", "warn", "literature-qa skill 目录未找到", "确认 agent_config_dir 配置正确") @@ -549,7 +438,7 @@ def _is_junction(path: Path) -> bool: try: import ctypes from ctypes import wintypes - FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 + FILE_ATTRIBUTE_REPARSE_POINT = 0x400 INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF GetFileAttributesW = ctypes.windll.kernel32.GetFileAttributesW @@ -569,36 +458,36 @@ def run_status(vault: Path, verbose: bool = False) -> int: cfg = load_vault_config(vault) config = load_domain_config(paths) ensure_base_views(vault, paths, config) - export_files = sorted(paths['exports'].glob('*.json')) - record_count = sum(1 for _ in paths['library_records'].rglob('*.md')) if paths['library_records'].exists() else 0 - note_count = sum(1 for _ in paths['literature'].rglob('*.md')) if paths['literature'].exists() else 0 - base_count = sum(1 for _ in paths['bases'].glob('*.base')) if paths['bases'].exists() else 0 + export_files = sorted(paths["exports"].glob("*.json")) + record_count = sum(1 for _ in paths["library_records"].rglob("*.md")) if paths["library_records"].exists() else 0 + note_count = sum(1 for _ in paths["literature"].rglob("*.md")) if paths["literature"].exists() else 0 + base_count = sum(1 for _ in paths["bases"].glob("*.base")) if paths["bases"].exists() else 0 ocr_done = 0 ocr_total = 0 - if paths['ocr'].exists(): - for meta_path in paths['ocr'].glob('*/meta.json'): + if paths["ocr"].exists(): + for meta_path in paths["ocr"].glob("*/meta.json"): ocr_total += 1 try: meta = read_json(meta_path) except Exception: continue - if str(meta.get('ocr_status', '')).strip().lower() == 'done': + if str(meta.get("ocr_status", "")).strip().lower() == "done": ocr_done += 1 - env_paths = [vault / '.env', paths['pipeline'] / '.env'] - env_found = [str(path.relative_to(vault)).replace('\\', '/') for path in env_paths if path.exists()] + env_paths = [vault / ".env", paths["pipeline"] / ".env"] + env_found = [str(path.relative_to(vault)).replace("\\", "/") for path in env_paths if path.exists()] # Count path errors path_error_count = 0 - if paths['library_records'].exists(): - for record_path in paths['library_records'].rglob('*.md'): + if paths["library_records"].exists(): + for record_path in paths["library_records"].rglob("*.md"): try: - text = record_path.read_text(encoding='utf-8') + text = record_path.read_text(encoding="utf-8") if re.search(r'^path_error:\s*"(.+?)"\s*$', text, re.MULTILINE): path_error_count += 1 except Exception: continue - print('PaperForge Lite status') + print("PaperForge Lite status") print(f"- vault: {vault}") print(f"- system_dir: {cfg['system_dir']}") print(f"- resources_dir: {cfg['resources_dir']}") @@ -616,6 +505,7 @@ def run_status(vault: Path, verbose: bool = False) -> int: print(f"- env: {', '.join(env_found) if env_found else 'not configured'}") return 0 + # ============================================================================= # Update 功能 # ============================================================================= @@ -624,5 +514,3 @@ GITHUB_REPO = "LLLin000/PaperForge" GITHUB_ZIP = f"https://github.com/{GITHUB_REPO}/archive/refs/heads/master.zip" UPDATEABLE_PATHS = ["skills", "pipeline", "command", "scripts"] - - diff --git a/paperforge/worker/sync.py b/paperforge/worker/sync.py index 3b04e955..b87f794c 100644 --- a/paperforge/worker/sync.py +++ b/paperforge/worker/sync.py @@ -1,147 +1,32 @@ from __future__ import annotations -import logging -import argparse -import csv -import hashlib + import html -import json +import logging import os import re -import shutil -import subprocess -import sys -import tempfile import urllib.parse -from json import JSONDecodeError -import zipfile from datetime import datetime, timezone from pathlib import Path from xml.etree import ElementTree as ET + import requests -import fitz -from PIL import Image + +from paperforge.config import load_vault_config, paperforge_paths +from paperforge.worker._utils import ( + _extract_year, + lookup_impact_factor, + read_json, + read_jsonl, + slugify_filename, + write_json, + write_jsonl, + yaml_block, + yaml_list, + yaml_quote, +) logger = logging.getLogger(__name__) -STANDARD_VIEW_NAMES = frozenset([ - "控制面板", "推荐分析", "待 OCR", "OCR 完成", - "待深度阅读", "深度阅读完成", "正式卡片", "全记录" -]) - -def load_simple_env(env_path: Path) -> None: - if not env_path.exists(): - return - for raw_line in env_path.read_text(encoding='utf-8').splitlines(): - line = raw_line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - key, value = line.split('=', 1) - key = key.strip() - if not key or key in os.environ: - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}): - value = value[1:-1] - os.environ[key] = value - -def read_json(path: Path): - return json.loads(path.read_text(encoding='utf-8')) -_JOURNAL_DB: dict[str, dict] | None = None - -def load_journal_db(vault: Path) -> dict[str, dict]: - """Load zoterostyle.json journal database.""" - global _JOURNAL_DB - if _JOURNAL_DB is not None: - return _JOURNAL_DB - zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json' - if zoterostyle_path.exists(): - try: - _JOURNAL_DB = read_json(zoterostyle_path) - except (JSONDecodeError, Exception): - _JOURNAL_DB = {} - else: - _JOURNAL_DB = {} - return _JOURNAL_DB - -def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str: - """Lookup impact factor: prefer zoterostyle.json, fallback to extra field.""" - if not journal_name: - return '' - journal_db = load_journal_db(vault) - if journal_name in journal_db: - rank_data = journal_db[journal_name].get('rank', {}) - if isinstance(rank_data, dict): - sciif = rank_data.get('sciif', '') - if sciif: - return str(sciif) - if extra: - if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra) - if if_match: - return if_match.group(1) - return '' - -def write_json(path: Path, data) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') - -def read_jsonl(path: Path): - rows = [] - if not path.exists(): - return rows - for line in path.read_text(encoding='utf-8').splitlines(): - line = line.strip() - if line: - rows.append(json.loads(line)) - return rows - -def write_jsonl(path: Path, rows) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows)) - if text: - text += '\n' - path.write_text(text, encoding='utf-8') - -def yaml_quote(value: str) -> str: - if isinstance(value, bool): - return 'true' if value else 'false' - return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"' - -def yaml_block(value: str) -> list[str]: - value = (value or '').strip() - if not value: - return ['abstract: |-', ' '] - lines = ['abstract: |-'] - for line in value.splitlines(): - lines.append(f' {line}') - return lines - -def yaml_list(key: str, values) -> list[str]: - cleaned = [str(value).strip() for value in values or [] if str(value).strip()] - if not cleaned: - return [f'{key}: []'] - lines = [f'{key}:'] - for value in cleaned: - lines.append(f' - {yaml_quote(value)}') - return lines - -def slugify_filename(text: str) -> str: - cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip() - return cleaned[:120] or 'untitled' - -def _extract_year(value: str) -> str: - match = re.search('(19|20)\\d{2}', value or '') - return match.group(0) if match else '' - - -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -149,14 +34,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -183,17 +61,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -204,6 +80,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + def build_collection_lookup(collections: dict) -> dict: path_cache = {} item_paths = {} @@ -212,233 +89,266 @@ def build_collection_lookup(collections: dict) -> dict: if key in path_cache: return path_cache[key] node = collections.get(key, {}) - parent = node.get('parent') or '' - name = node.get('name', '') - parent_path = path_for(parent) if parent else '' - full_path = f'{parent_path}/{name}' if parent_path else name + parent = node.get("parent") or "" + name = node.get("name", "") + parent_path = path_for(parent) if parent else "" + full_path = f"{parent_path}/{name}" if parent_path else name path_cache[key] = full_path return full_path + for key, node in collections.items(): full_path = path_for(key) - for item_id in node.get('items', []): + for item_id in node.get("items", []): item_paths.setdefault(item_id, []).append(full_path) - return {'path_by_key': path_cache, 'paths_by_item_id': item_paths} + return {"path_by_key": path_cache, "paths_by_item_id": item_paths} + def export_collection_paths(export_path: Path) -> list[str]: data = read_json(export_path) if not isinstance(data, dict): return [] - collections = data.get('collections', {}) + collections = data.get("collections", {}) if not isinstance(collections, dict): return [] lookup = build_collection_lookup(collections) - paths = sorted({path for path in lookup.get('path_by_key', {}).values() if str(path or '').strip()}, key=lambda value: (value.count('/'), value)) + paths = sorted( + {path for path in lookup.get("path_by_key", {}).values() if str(path or "").strip()}, + key=lambda value: (value.count("/"), value), + ) return paths + def load_domain_collection_catalog(paths: dict[str, Path]) -> dict[str, list[str]]: - config_path = paths['config'] - config = read_json(config_path) if config_path.exists() else {'domains': []} - domain_entries = list(config.get('domains', [])) - entry_by_export = {entry.get('export_file', ''): dict(entry) for entry in domain_entries if entry.get('export_file')} - export_files = sorted(paths['exports'].glob('*.json')) + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} + domain_entries = list(config.get("domains", [])) + entry_by_export = { + entry.get("export_file", ""): dict(entry) for entry in domain_entries if entry.get("export_file") + } + export_files = sorted(paths["exports"].glob("*.json")) changed = False for export_path in export_files: entry = entry_by_export.get(export_path.name) if not entry: - entry = {'domain': export_path.stem, 'export_file': export_path.name, 'allowed_collections': []} + entry = {"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []} entry_by_export[export_path.name] = entry changed = True derived = export_collection_paths(export_path) - if entry.get('allowed_collections', []) != derived: - entry['allowed_collections'] = derived + if entry.get("allowed_collections", []) != derived: + entry["allowed_collections"] = derived changed = True - domains = sorted(entry_by_export.values(), key=lambda entry: entry.get('domain', '')) - if changed or config.get('domains', []) != domains: - write_json(config_path, {'domains': domains}) - return {entry.get('domain', ''): entry.get('allowed_collections', []) for entry in domains if entry.get('domain')} + domains = sorted(entry_by_export.values(), key=lambda entry: entry.get("domain", "")) + if changed or config.get("domains", []) != domains: + write_json(config_path, {"domains": domains}) + return {entry.get("domain", ""): entry.get("allowed_collections", []) for entry in domains if entry.get("domain")} + def load_export_inventory(paths: dict[str, Path]) -> dict[str, dict]: - inventory = {'doi': {}, 'pmid': {}, 'title': {}} - for export_path in sorted(paths['exports'].glob('*.json')): + inventory = {"doi": {}, "pmid": {}, "title": {}} + for export_path in sorted(paths["exports"].glob("*.json")): domain = export_path.stem for item in load_export_rows(export_path): - record = {'zotero_key': item.get('key', ''), 'domain': domain, 'title': item.get('title', ''), 'doi': item.get('doi', ''), 'pmid': item.get('pmid', ''), 'collections': item.get('collections', [])} - doi = str(record.get('doi', '') or '').strip().lower() - pmid = str(record.get('pmid', '') or '').strip() - title = normalize_candidate_title(record.get('title', '')) - if doi and doi not in inventory['doi']: - inventory['doi'][doi] = record - if pmid and pmid not in inventory['pmid']: - inventory['pmid'][pmid] = record - if title and title not in inventory['title']: - inventory['title'][title] = record + record = { + "zotero_key": item.get("key", ""), + "domain": domain, + "title": item.get("title", ""), + "doi": item.get("doi", ""), + "pmid": item.get("pmid", ""), + "collections": item.get("collections", []), + } + doi = str(record.get("doi", "") or "").strip().lower() + pmid = str(record.get("pmid", "") or "").strip() + title = normalize_candidate_title(record.get("title", "")) + if doi and doi not in inventory["doi"]: + inventory["doi"][doi] = record + if pmid and pmid not in inventory["pmid"]: + inventory["pmid"][pmid] = record + if title and title not in inventory["title"]: + inventory["title"][title] = record return inventory + def find_existing_library_match(row: dict, inventory: dict[str, dict]) -> dict | None: - doi = str(row.get('doi', '') or '').strip().lower() - if doi and doi in inventory['doi']: - return inventory['doi'][doi] - pmid = str(row.get('pmid', '') or '').strip() - if pmid and pmid in inventory['pmid']: - return inventory['pmid'][pmid] - title = normalize_candidate_title(row.get('title', '')) - if title and title in inventory['title']: - return inventory['title'][title] + doi = str(row.get("doi", "") or "").strip().lower() + if doi and doi in inventory["doi"]: + return inventory["doi"][doi] + pmid = str(row.get("pmid", "") or "").strip() + if pmid and pmid in inventory["pmid"]: + return inventory["pmid"][pmid] + title = normalize_candidate_title(row.get("title", "")) + if title and title in inventory["title"]: + return inventory["title"][title] return None + def resolve_collection_choice(domain: str, raw_value: str, catalog: dict[str, list[str]]) -> dict[str, str]: - text = str(raw_value or '').strip() + text = str(raw_value or "").strip() if not text: - return {'resolved': '', 'match': '', 'input': ''} + return {"resolved": "", "match": "", "input": ""} allowed = [path for path in catalog.get(domain, []) if path] if not allowed: - return {'resolved': '', 'match': 'no_catalog', 'input': text} + return {"resolved": "", "match": "no_catalog", "input": text} lower_text = text.lower() - if '/' not in text: - leaf_matches = [path for path in allowed if path.split('/')[-1].strip().lower() == lower_text] + if "/" not in text: + leaf_matches = [path for path in allowed if path.split("/")[-1].strip().lower() == lower_text] leaf_matches = sorted(set(leaf_matches)) if len(leaf_matches) > 1: - return {'resolved': '', 'match': 'ambiguous_leaf', 'input': text} + return {"resolved": "", "match": "ambiguous_leaf", "input": text} exact_map = {path: path for path in allowed} if text in exact_map: - return {'resolved': text, 'match': 'exact', 'input': text} + return {"resolved": text, "match": "exact", "input": text} lower_exact = {path.lower(): path for path in allowed} if lower_text in lower_exact: - return {'resolved': lower_exact[lower_text], 'match': 'exact_ci', 'input': text} - leaf_matches = [path for path in allowed if path.split('/')[-1].strip().lower() == lower_text] + return {"resolved": lower_exact[lower_text], "match": "exact_ci", "input": text} + leaf_matches = [path for path in allowed if path.split("/")[-1].strip().lower() == lower_text] if len(leaf_matches) == 1: - return {'resolved': leaf_matches[0], 'match': 'leaf', 'input': text} - suffix_matches = [path for path in allowed if path.lower().endswith('/' + lower_text) or path.lower() == lower_text] + return {"resolved": leaf_matches[0], "match": "leaf", "input": text} + suffix_matches = [path for path in allowed if path.lower().endswith("/" + lower_text) or path.lower() == lower_text] suffix_matches = sorted(set(suffix_matches)) if len(suffix_matches) == 1: - return {'resolved': suffix_matches[0], 'match': 'suffix', 'input': text} - compact = re.sub('\\s+', '', lower_text) + return {"resolved": suffix_matches[0], "match": "suffix", "input": text} + compact = re.sub("\\s+", "", lower_text) compact_matches = [] for path in allowed: - path_compact = re.sub('\\s+', '', path.lower()) - if path_compact.endswith('/' + compact) or path_compact == compact: + path_compact = re.sub("\\s+", "", path.lower()) + if path_compact.endswith("/" + compact) or path_compact == compact: compact_matches.append(path) compact_matches = sorted(set(compact_matches)) if len(compact_matches) == 1: - return {'resolved': compact_matches[0], 'match': 'compact_suffix', 'input': text} - match = 'ambiguous' if leaf_matches or suffix_matches or compact_matches else 'unresolved' - return {'resolved': '', 'match': match, 'input': text} + return {"resolved": compact_matches[0], "match": "compact_suffix", "input": text} + match = "ambiguous" if leaf_matches or suffix_matches or compact_matches else "unresolved" + return {"resolved": "", "match": match, "input": text} + def apply_candidate_collection_resolution(row: dict, catalog: dict[str, list[str]]) -> dict: resolved = dict(row) - domain = str(resolved.get('domain', '') or '').strip() - recommended = resolve_collection_choice(domain, resolved.get('recommended_collection', ''), catalog) - user = resolve_collection_choice(domain, resolved.get('user_collection', ''), catalog) - resolved['recommended_collection'] = recommended.get('resolved', '') - resolved['user_collection_resolved'] = user.get('resolved', '') - if str(resolved.get('user_collection', '') or '').strip(): - resolved['final_collection'] = user.get('resolved', '') - resolved['collection_resolution'] = f"user_{user.get('match', 'unresolved')}" if user.get('match') else 'user_unresolved' + domain = str(resolved.get("domain", "") or "").strip() + recommended = resolve_collection_choice(domain, resolved.get("recommended_collection", ""), catalog) + user = resolve_collection_choice(domain, resolved.get("user_collection", ""), catalog) + resolved["recommended_collection"] = recommended.get("resolved", "") + resolved["user_collection_resolved"] = user.get("resolved", "") + if str(resolved.get("user_collection", "") or "").strip(): + resolved["final_collection"] = user.get("resolved", "") + resolved["collection_resolution"] = ( + f"user_{user.get('match', 'unresolved')}" if user.get("match") else "user_unresolved" + ) else: - resolved['final_collection'] = recommended.get('resolved', '') - resolved['collection_resolution'] = f"recommended_{recommended.get('match', 'unresolved')}" if recommended.get('match') else 'recommended_unresolved' + resolved["final_collection"] = recommended.get("resolved", "") + resolved["collection_resolution"] = ( + f"recommended_{recommended.get('match', 'unresolved')}" + if recommended.get("match") + else "recommended_unresolved" + ) return resolved + def apply_existing_library_match(row: dict, inventory: dict[str, dict]) -> dict: resolved = dict(row) match = find_existing_library_match(resolved, inventory) if not match: - resolved['existing_zotero_key'] = '' - resolved['existing_collections'] = [] - resolved['duplicate_hint'] = '' + resolved["existing_zotero_key"] = "" + resolved["existing_collections"] = [] + resolved["duplicate_hint"] = "" return resolved - resolved['existing_zotero_key'] = str(match.get('zotero_key', '') or '').strip() - resolved['existing_collections'] = list(match.get('collections', []) or []) - collections_text = ' | '.join(resolved['existing_collections']) + resolved["existing_zotero_key"] = str(match.get("zotero_key", "") or "").strip() + resolved["existing_collections"] = list(match.get("collections", []) or []) + collections_text = " | ".join(resolved["existing_collections"]) if collections_text: - resolved['duplicate_hint'] = f"已存在于 Zotero: {resolved['existing_zotero_key']} ({collections_text})" + resolved["duplicate_hint"] = f"已存在于 Zotero: {resolved['existing_zotero_key']} ({collections_text})" else: - resolved['duplicate_hint'] = f"已存在于 Zotero: {resolved['existing_zotero_key']}" + resolved["duplicate_hint"] = f"已存在于 Zotero: {resolved['existing_zotero_key']}" return resolved + def resolve_item_collection_paths(item: dict, collection_lookup: dict) -> list[str]: paths = [] - collection_keys = item.get('collections') or [] + collection_keys = item.get("collections") or [] if collection_keys: for key in collection_keys: - paths.append(collection_lookup.get('path_by_key', {}).get(key, key)) - item_id = item.get('itemID') + paths.append(collection_lookup.get("path_by_key", {}).get(key, key)) + item_id = item.get("itemID") if item_id is not None: - paths.extend(collection_lookup.get('paths_by_item_id', {}).get(item_id, [])) - return sorted({path for path in paths if path}, key=lambda value: (-value.count('/'), value)) + paths.extend(collection_lookup.get("paths_by_item_id", {}).get(item_id, [])) + return sorted({path for path in paths if path}, key=lambda value: (-value.count("/"), value)) + def obsidian_wikilink_for_pdf(pdf_path: str, vault_dir: Path, zotero_dir: Path | None = None) -> str: - text = str(pdf_path or '').strip() + text = str(pdf_path or "").strip() if not text: - return '' + return "" # Handle storage: prefix paths by resolving through zotero_dir - if text.startswith('storage:') and zotero_dir is not None: - storage_rel = text[len('storage:'):].lstrip('/').lstrip('\\') - absolute_pdf_path = (zotero_dir / storage_rel.replace('/', os.sep)).resolve() + if text.startswith("storage:") and zotero_dir is not None: + storage_rel = text[len("storage:") :].lstrip("/").lstrip("\\") + absolute_pdf_path = (zotero_dir / storage_rel.replace("/", os.sep)).resolve() absolute_str = str(absolute_pdf_path) else: absolute_str = absolutize_vault_path(vault_dir, text, resolve_junction=True) if not absolute_str: - return '' + return "" absolute_path = Path(absolute_str) try: relative = absolute_path.relative_to(vault_dir) except ValueError: - return f'[[{absolute_path.as_posix()}]]' - return f'[[{relative.as_posix()}]]' + return f"[[{absolute_path.as_posix()}]]" + return f"[[{relative.as_posix()}]]" + def absolutize_vault_path(vault: Path, path: str, resolve_junction: bool = False) -> str: - text = str(path or '').strip() + text = str(path or "").strip() if not text: - return '' + return "" candidate = Path(text) - if candidate.is_absolute(): - result = str(candidate) - else: - result = str((vault / text.replace('/', os.sep)).resolve()) + result = str(candidate) if candidate.is_absolute() else str((vault / text.replace("/", os.sep)).resolve()) if resolve_junction: from paperforge.pdf_resolver import resolve_junction + result = str(resolve_junction(Path(result))) return result + def obsidian_wikilink_for_path(vault: Path, path: str) -> str: absolute = absolutize_vault_path(vault, path, resolve_junction=True) if not absolute: - return '' + return "" absolute_path = Path(absolute) try: relative = absolute_path.relative_to(vault) except ValueError: - return f'[[{absolute_path.as_posix()}]]' - return f'[[{relative.as_posix()}]]' + return f"[[{absolute_path.as_posix()}]]" + return f"[[{relative.as_posix()}]]" + def collection_fields(collection_paths: list[str]) -> dict[str, str | list[str]]: paths = [path for path in collection_paths if path] - primary = paths[0] if paths else '' + primary = paths[0] if paths else "" if paths: - primary = sorted(paths, key=lambda value: (value.count('/'), len(value), value), reverse=True)[0] + primary = sorted(paths, key=lambda value: (value.count("/"), len(value), value), reverse=True)[0] tags = [] seen = set() for path in paths: - for part in [segment.strip() for segment in path.split('/') if segment.strip()]: + for part in [segment.strip() for segment in path.split("/") if segment.strip()]: if part not in seen: seen.add(part) tags.append(part) group = primary - return {'collections': paths, 'collection_tags': tags, 'collection_group': [group] if group else []} + return {"collections": paths, "collection_tags": tags, "collection_group": [group] if group else []} + def extract_authors(item: dict) -> list[str]: authors = [] - for creator in item.get('creators', []): - if creator.get('creatorType') != 'author': + for creator in item.get("creators", []): + if creator.get("creatorType") != "author": continue - full_name = ' '.join((part for part in [creator.get('firstName', ''), creator.get('lastName', '')] if part)).strip() + full_name = " ".join( + part for part in [creator.get("firstName", ""), creator.get("lastName", "")] if part + ).strip() if full_name: authors.append(full_name) - elif creator.get('name'): - authors.append(creator['name']) + elif creator.get("name"): + authors.append(creator["name"]) return authors + def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tuple[str, str, str]: """Normalize a BBT attachment path to a consistent storage: format. @@ -459,41 +369,40 @@ def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tup Zotero storage paths. bbt_path_raw preserves the original input for debugging. zotero_storage_key is the 8-character Zotero key. """ - raw = str(path or '').strip() + raw = str(path or "").strip() if not raw: - return ('', '', '') + return ("", "", "") bbt_path_raw = raw # Format 2: Already has storage: prefix — pass through with slash normalization - if raw.startswith('storage:'): - storage_rel = raw[len('storage:'):].lstrip('/').lstrip('\\') - storage_rel = storage_rel.replace('\\', '/') - parts = storage_rel.split('/') - zotero_storage_key = parts[0] if parts else '' - return (f'storage:{storage_rel}', bbt_path_raw, zotero_storage_key) + if raw.startswith("storage:"): + storage_rel = raw[len("storage:") :].lstrip("/").lstrip("\\") + storage_rel = storage_rel.replace("\\", "/") + parts = storage_rel.split("/") + zotero_storage_key = parts[0] if parts else "" + return (f"storage:{storage_rel}", bbt_path_raw, zotero_storage_key) # Format 1: Absolute Windows path pointing to Zotero storage candidate = Path(raw) if candidate.is_absolute(): - norm_path = raw.replace('\\', '/') + norm_path = raw.replace("\\", "/") # Detect Zotero storage pattern: .../storage/8CHARKEY/... - if '/storage/' in norm_path: - parts_after_storage = norm_path.split('/storage/', 1)[1] - parts = parts_after_storage.split('/') + if "/storage/" in norm_path: + parts_after_storage = norm_path.split("/storage/", 1)[1] + parts = parts_after_storage.split("/") if len(parts) >= 2 and len(parts[0]) == 8 and parts[0].isalnum(): zotero_storage_key = parts[0] - filename = '/'.join(parts[1:]) - return (f'storage:{zotero_storage_key}/{filename}', - bbt_path_raw, zotero_storage_key) + filename = "/".join(parts[1:]) + return (f"storage:{zotero_storage_key}/{filename}", bbt_path_raw, zotero_storage_key) # Absolute path but not in Zotero storage — mark as absolute - return (f'absolute:{raw}', bbt_path_raw, '') + return (f"absolute:{raw}", bbt_path_raw, "") # Format 3: Bare relative path — prepend storage: prefix - norm = raw.replace('\\', '/') - parts = norm.split('/') - zotero_storage_key = parts[0] if parts else '' - return (f'storage:{norm}', bbt_path_raw, zotero_storage_key) + norm = raw.replace("\\", "/") + parts = norm.split("/") + zotero_storage_key = parts[0] if parts else "" + return (f"storage:{norm}", bbt_path_raw, zotero_storage_key) def _identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict]]: @@ -512,32 +421,28 @@ def _identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict] main_pdf_attachment may be None if no PDFs found. supplementary_attachments is a list of all other PDF attachments. """ - pdf_attachments = [ - a for a in attachments - if isinstance(a, dict) and a.get('contentType') == 'application/pdf' - ] + pdf_attachments = [a for a in attachments if isinstance(a, dict) and a.get("contentType") == "application/pdf"] if not pdf_attachments: return (None, []) # Priority 1: Title exactly equals "PDF" for att in pdf_attachments: - if att.get('title') == 'PDF': + if att.get("title") == "PDF": main = att supplementary = [a for a in pdf_attachments if a is not main] return (main, supplementary) # Priority 2: Largest file by size (if size field is available and differentiated) - sized = [(a, a.get('size', 0) or 0) for a in pdf_attachments] + sized = [(a, a.get("size", 0) or 0) for a in pdf_attachments] sized.sort(key=lambda x: x[1], reverse=True) - if sized and sized[0][1] > 0: - if len(sized) == 1 or sized[0][1] > sized[1][1]: - main = sized[0][0] - supplementary = [a for a in pdf_attachments if a is not main] - return (main, supplementary) + if sized and sized[0][1] > 0 and (len(sized) == 1 or sized[0][1] > sized[1][1]): + main = sized[0][0] + supplementary = [a for a in pdf_attachments if a is not main] + return (main, supplementary) # Priority 2b (sizes equal or unavailable): shortest title - titled = [(a, len(str(a.get('title', '')))) for a in pdf_attachments] + titled = [(a, len(str(a.get("title", "")))) for a in pdf_attachments] titled.sort(key=lambda x: x[1]) main = titled[0][0] supplementary = [a for a in pdf_attachments if a is not main] @@ -548,122 +453,213 @@ def load_export_rows(path: Path) -> list[dict]: data = read_json(path) if isinstance(data, list): return data - if isinstance(data, dict) and isinstance(data.get('items'), list): - collection_lookup = build_collection_lookup(data.get('collections', {})) + if isinstance(data, dict) and isinstance(data.get("items"), list): + collection_lookup = build_collection_lookup(data.get("collections", {})) rows = [] - for item in data['items']: - if item.get('itemType') in {'attachment', 'note', 'annotation'}: + for item in data["items"]: + if item.get("itemType") in {"attachment", "note", "annotation"}: continue attachments = [] - for attachment in item.get('attachments', []): + for attachment in item.get("attachments", []): if not isinstance(attachment, dict): continue - raw_path = attachment.get('path', '') + raw_path = attachment.get("path", "") normalized_path, bbt_path_raw, zotero_storage_key = _normalize_attachment_path(raw_path) # Preserve contentType from BBT if present; fallback to file extension - content_type = attachment.get('contentType', '') - if not content_type and str(normalized_path).lower().endswith('.pdf'): - content_type = 'application/pdf' - attachments.append({ - 'path': normalized_path, - 'contentType': content_type, - 'title': attachment.get('title', ''), - 'bbt_path_raw': bbt_path_raw, - 'zotero_storage_key': zotero_storage_key, - 'size': attachment.get('size', 0) or 0, - }) + content_type = attachment.get("contentType", "") + if not content_type and str(normalized_path).lower().endswith(".pdf"): + content_type = "application/pdf" + attachments.append( + { + "path": normalized_path, + "contentType": content_type, + "title": attachment.get("title", ""), + "bbt_path_raw": bbt_path_raw, + "zotero_storage_key": zotero_storage_key, + "size": attachment.get("size", 0) or 0, + } + ) main_pdf, supplementary_pdfs = _identify_main_pdf(attachments) - pdf_path = main_pdf['path'] if main_pdf else '' - bbt_path_raw = main_pdf['bbt_path_raw'] if main_pdf else '' - zotero_storage_key = main_pdf['zotero_storage_key'] if main_pdf else '' - path_error = 'not_found' if not main_pdf else '' - supplementary = [a['path'] for a in supplementary_pdfs] if supplementary_pdfs else [] + pdf_path = main_pdf["path"] if main_pdf else "" + bbt_path_raw = main_pdf["bbt_path_raw"] if main_pdf else "" + zotero_storage_key = main_pdf["zotero_storage_key"] if main_pdf else "" + path_error = "not_found" if not main_pdf else "" + supplementary = [a["path"] for a in supplementary_pdfs] if supplementary_pdfs else [] attachment_count = len(attachments) - rows.append({ - 'key': item.get('key') or item.get('itemKey', ''), - 'title': item.get('title', ''), - 'authors': extract_authors(item), - 'abstract': item.get('abstractNote', ''), - 'journal': item.get('publicationTitle', ''), - 'year': _extract_year(item.get('date', '')), - 'date': item.get('date', ''), - 'doi': item.get('DOI', ''), - 'pmid': item.get('PMID', ''), - 'collections': resolve_item_collection_paths(item, collection_lookup), - 'attachments': attachments, - 'pdf_path': pdf_path, - 'supplementary': supplementary, - 'attachment_count': attachment_count, - 'bbt_path_raw': bbt_path_raw, - 'zotero_storage_key': zotero_storage_key, - 'path_error': path_error, - }) + rows.append( + { + "key": item.get("key") or item.get("itemKey", ""), + "title": item.get("title", ""), + "authors": extract_authors(item), + "abstract": item.get("abstractNote", ""), + "journal": item.get("publicationTitle", ""), + "year": _extract_year(item.get("date", "")), + "date": item.get("date", ""), + "doi": item.get("DOI", ""), + "pmid": item.get("PMID", ""), + "collections": resolve_item_collection_paths(item, collection_lookup), + "attachments": attachments, + "pdf_path": pdf_path, + "supplementary": supplementary, + "attachment_count": attachment_count, + "bbt_path_raw": bbt_path_raw, + "zotero_storage_key": zotero_storage_key, + "path_error": path_error, + } + ) return rows - raise ValueError(f'Unsupported export format: {path}') + raise ValueError(f"Unsupported export format: {path}") + def compute_final_collection(row: dict) -> str: - user_raw = str(row.get('user_collection', '') or '').strip() - user_resolved = str(row.get('user_collection_resolved', '') or '').strip() - recommended = str(row.get('recommended_collection', '') or '').strip() + user_raw = str(row.get("user_collection", "") or "").strip() + user_resolved = str(row.get("user_collection_resolved", "") or "").strip() + recommended = str(row.get("recommended_collection", "") or "").strip() if user_raw: return user_resolved return recommended + def canonicalize_decision(value: str) -> str: - text = str(value or '').strip() - if text in {'', '待查'}: - return '待定' - if text in {'排除', '不纳入'}: - return '不纳入' - if text == '纳入': - return '纳入' - return '待定' + text = str(value or "").strip() + if text in {"", "待查"}: + return "待定" + if text in {"排除", "不纳入"}: + return "不纳入" + if text == "纳入": + return "纳入" + return "待定" + def candidate_markdown(row: dict) -> str: row = dict(row) - row['final_collection'] = compute_final_collection(row) - row['decision'] = canonicalize_decision(row.get('decision', '')) - lines = ['---'] - ordered_keys = ['candidate_id', 'domain', 'title', 'authors', 'year', 'journal', 'doi', 'pmid', 'source', 'requester_skill', 'request_context', 'abstract_short', 'decision', 'recommended_collection', 'recommend_confidence', 'recommend_reason', 'user_collection', 'user_collection_resolved', 'final_collection', 'collection_resolution', 'duplicate_hint', 'existing_zotero_key', 'existing_collections', 'import_status', 'note', 'candidate_source_type', 'source_zotero_key', 'cited_ref_number', 'trigger_sentence', 'source_context', 'task_relevance_reason', 'harvest_priority', 'raw_reference', 'status'] - row.setdefault('status', 'candidate') + row["final_collection"] = compute_final_collection(row) + row["decision"] = canonicalize_decision(row.get("decision", "")) + lines = ["---"] + ordered_keys = [ + "candidate_id", + "domain", + "title", + "authors", + "year", + "journal", + "doi", + "pmid", + "source", + "requester_skill", + "request_context", + "abstract_short", + "decision", + "recommended_collection", + "recommend_confidence", + "recommend_reason", + "user_collection", + "user_collection_resolved", + "final_collection", + "collection_resolution", + "duplicate_hint", + "existing_zotero_key", + "existing_collections", + "import_status", + "note", + "candidate_source_type", + "source_zotero_key", + "cited_ref_number", + "trigger_sentence", + "source_context", + "task_relevance_reason", + "harvest_priority", + "raw_reference", + "status", + ] + row.setdefault("status", "candidate") for key in ordered_keys: - value = row.get(key, '') + value = row.get(key, "") if isinstance(value, list): - lines.append(f'{key}:') + lines.append(f"{key}:") for item in value: - lines.append(f' - {yaml_quote(item)}') - elif value == '': - lines.append(f'{key}:') - elif '\n' in str(value): - lines.extend(yaml_block(str(value)).copy() if key == 'abstract' else [f'{key}: |-'] + [f' {line}' for line in str(value).splitlines()]) + lines.append(f" - {yaml_quote(item)}") + elif value == "": + lines.append(f"{key}:") + elif "\n" in str(value): + lines.extend( + yaml_block(str(value)).copy() + if key == "abstract" + else [f"{key}: |-"] + [f" {line}" for line in str(value).splitlines()] + ) else: - lines.append(f'{key}: {yaml_quote(value)}') - lines.extend(['---', '', f"# {row['candidate_id']}", '', '候选文献轻量记录,仅用于 Base 决策和 write-back 触发,不是正式文献卡片。', '']) - return '\n'.join(lines) + lines.append(f"{key}: {yaml_quote(value)}") + lines.extend( + [ + "---", + "", + f"# {row['candidate_id']}", + "", + "候选文献轻量记录,仅用于 Base 决策和 write-back 触发,不是正式文献卡片。", + "", + ] + ) + return "\n".join(lines) + def generate_review(candidates: list[dict]) -> str: normalized = [] for row in candidates: copy = dict(row) - copy['decision'] = canonicalize_decision(copy.get('decision', '')) + copy["decision"] = canonicalize_decision(copy.get("decision", "")) normalized.append(copy) - include = [c for c in normalized if c.get('decision') == '纳入'] - exclude = [c for c in normalized if c.get('decision') == '不纳入'] - lines = ['# 本轮候选总览', '', '## 检索背景', '', f'- 候选数量:{len(normalized)}', f'- 建议纳入:{len(include)}', f'- 不纳入:{len(exclude)}', '', '## 总体判断', '', '- 当前候选池已经按决策状态分层,可直接进入 Base 处理。', '', '## 推荐优先纳入', ''] + include = [c for c in normalized if c.get("decision") == "纳入"] + exclude = [c for c in normalized if c.get("decision") == "不纳入"] + lines = [ + "# 本轮候选总览", + "", + "## 检索背景", + "", + f"- 候选数量:{len(normalized)}", + f"- 建议纳入:{len(include)}", + f"- 不纳入:{len(exclude)}", + "", + "## 总体判断", + "", + "- 当前候选池已经按决策状态分层,可直接进入 Base 处理。", + "", + "## 推荐优先纳入", + "", + ] if include: for row in include: - lines.extend([f"### {row['candidate_id']}", '', f"- 标题:{row['title']}", f'- 推荐分类:`{compute_final_collection(row)}`', f"- 理由:{row.get('recommend_reason', '')}", '']) + lines.extend( + [ + f"### {row['candidate_id']}", + "", + f"- 标题:{row['title']}", + f"- 推荐分类:`{compute_final_collection(row)}`", + f"- 理由:{row.get('recommend_reason', '')}", + "", + ] + ) else: - lines.extend(['- 暂无', '']) - lines.extend(['## 不纳入', '']) + lines.extend(["- 暂无", ""]) + lines.extend(["## 不纳入", ""]) if exclude: for row in exclude: - lines.extend([f"### {row['candidate_id']}", '', f"- 标题:{row['title']}", f"- 理由:{row.get('recommend_reason', '')}", '']) + lines.extend( + [ + f"### {row['candidate_id']}", + "", + f"- 标题:{row['title']}", + f"- 理由:{row.get('recommend_reason', '')}", + "", + ] + ) else: - lines.extend(['- 暂无', '']) - lines.extend(['## 下一步', '', '1. 在 Base 中确认决策。', '2. 对纳入项执行 write-back。', '3. 刷新正式索引。', '']) - return '\n'.join(lines) -DEEP_READING_HEADER = '## 🔍 精读' + lines.extend(["- 暂无", ""]) + lines.extend(["## 下一步", "", "1. 在 Base 中确认决策。", "2. 对纳入项执行 write-back。", "3. 刷新正式索引。", ""]) + return "\n".join(lines) + + +DEEP_READING_HEADER = "## 🔍 精读" + def extract_preserved_deep_reading(text: str) -> str: """Extract the `## 🔍 精读` section by matching it as a real markdown header. @@ -672,14 +668,15 @@ def extract_preserved_deep_reading(text: str) -> str: avoiding false positives from prose text that merely mentions the string. """ if not text: - return '' - match = re.search('^## 🔍 精读\\s*$', text, re.MULTILINE) + return "" + match = re.search("^## 🔍 精读\\s*$", text, re.MULTILINE) if not match: - return '' + return "" start = match.start() preserved = text[start:].strip() return preserved + def has_deep_reading_content(text: str) -> bool: """Return True only if the deep-reading section contains *substantive* content. @@ -691,7 +688,7 @@ def has_deep_reading_content(text: str) -> bool: preserved = extract_preserved_deep_reading(text) if not preserved: return False - body = preserved.replace(DEEP_READING_HEADER, '').strip() + body = preserved.replace(DEEP_READING_HEADER, "").strip() if not body: return False lines = body.splitlines() @@ -701,192 +698,275 @@ def has_deep_reading_content(text: str) -> bool: stripped = line.strip() if not stripped: continue - if stripped.startswith('### '): + if stripped.startswith("### "): continue - if re.match('^>\\s*\\[!', stripped): + if re.match("^>\\s*\\[!", stripped): continue - if '(待补充)' in stripped: + if "(待补充)" in stripped: continue - if re.match('^[-*]\\s*$', stripped): + if re.match("^[-*]\\s*$", stripped): continue non_placeholder_chars += len(stripped) - if re.search('[\\u4e00-\\u9fff]', stripped) and re.search('[。!?\\.\\!\\?]$', stripped): + if re.search("[\\u4e00-\\u9fff]", stripped) and re.search("[。!?\\.\\!\\?]$", stripped): has_prose_sentence = True return has_prose_sentence or non_placeholder_chars >= 20 + def library_record_markdown(row: dict) -> str: - lines = ['---', f"zotero_key: {row.get('zotero_key', '')}", f"domain: {row.get('domain', '')}", f"title: {yaml_quote(row.get('title', ''))}", f"year: {row.get('year', '')}", f"doi: {yaml_quote(row.get('doi', ''))}", f"date: {yaml_quote(row.get('date', ''))}", f"collection_path: {yaml_quote(row.get('collection_path', ''))}", f"has_pdf: {('true' if row.get('has_pdf') else 'false')}", f"pdf_path: {yaml_quote(row.get('pdf_path', ''))}", f"bbt_path_raw: {yaml_quote(row.get('bbt_path_raw', ''))}", f"zotero_storage_key: {yaml_quote(row.get('zotero_storage_key', ''))}", f"attachment_count: {row.get('attachment_count', 0)}"] + lines = [ + "---", + f"zotero_key: {row.get('zotero_key', '')}", + f"domain: {row.get('domain', '')}", + f"title: {yaml_quote(row.get('title', ''))}", + f"year: {row.get('year', '')}", + f"doi: {yaml_quote(row.get('doi', ''))}", + f"date: {yaml_quote(row.get('date', ''))}", + f"collection_path: {yaml_quote(row.get('collection_path', ''))}", + f"has_pdf: {('true' if row.get('has_pdf') else 'false')}", + f"pdf_path: {yaml_quote(row.get('pdf_path', ''))}", + f"bbt_path_raw: {yaml_quote(row.get('bbt_path_raw', ''))}", + f"zotero_storage_key: {yaml_quote(row.get('zotero_storage_key', ''))}", + f"attachment_count: {row.get('attachment_count', 0)}", + ] # supplementary as YAML list of wikilinks (already formatted by caller) - supplementary = row.get('supplementary', []) + supplementary = row.get("supplementary", []) if supplementary: - lines.append('supplementary:') + lines.append("supplementary:") for wikilink in supplementary: - lines.append(f' - {yaml_quote(wikilink)}') + lines.append(f" - {yaml_quote(wikilink)}") else: - lines.append('supplementary: []') + lines.append("supplementary: []") # path_error only emitted when there is an actual error - if row.get('path_error'): + if row.get("path_error"): lines.append(f"path_error: {yaml_quote(row.get('path_error', ''))}") - lines.extend([ - f"fulltext_md_path: {yaml_quote(row.get('fulltext_md_path', ''))}", - f"recommend_analyze: {('true' if row.get('recommend_analyze') else 'false')}", - f"analyze: {('true' if row.get('analyze') else 'false')}", - f"do_ocr: {('true' if row.get('do_ocr') else 'false')}", - f"ocr_status: {yaml_quote(row.get('ocr_status', 'pending'))}", - f"deep_reading_status: {yaml_quote(row.get('deep_reading_status', 'pending'))}", - f"analysis_note: {yaml_quote(row.get('analysis_note', ''))}", - ]) - lines.extend(yaml_list('collection_group', row.get('collection_group', []))) - lines.extend(yaml_list('collections', row.get('collections', []))) - lines.extend(yaml_list('collection_tags', row.get('collection_tags', []))) + lines.extend( + [ + f"fulltext_md_path: {yaml_quote(row.get('fulltext_md_path', ''))}", + f"recommend_analyze: {('true' if row.get('recommend_analyze') else 'false')}", + f"analyze: {('true' if row.get('analyze') else 'false')}", + f"do_ocr: {('true' if row.get('do_ocr') else 'false')}", + f"ocr_status: {yaml_quote(row.get('ocr_status', 'pending'))}", + f"deep_reading_status: {yaml_quote(row.get('deep_reading_status', 'pending'))}", + f"analysis_note: {yaml_quote(row.get('analysis_note', ''))}", + ] + ) + lines.extend(yaml_list("collection_group", row.get("collection_group", []))) + lines.extend(yaml_list("collections", row.get("collections", []))) + lines.extend(yaml_list("collection_tags", row.get("collection_tags", []))) lines.append(f"first_author: {yaml_quote(row.get('first_author', ''))}") lines.append(f"journal: {yaml_quote(row.get('journal', ''))}") lines.append(f"impact_factor: {yaml_quote(row.get('impact_factor', ''))}") - lines.extend(['---', '', f"# {row.get('title', '')}", '', '正式库控制记录。', '', '- `recommend_analyze` 仅由 `has_pdf=true` 推导。', '- `analyze` 控制是否生成正式文献卡片。', '- `do_ocr` 控制 OCR 任务。', '- `deep_reading_status` 仅两级:`pending`(未精读)/ `done`(已精读)。', '']) - return '\n'.join(lines) + lines.extend( + [ + "---", + "", + f"# {row.get('title', '')}", + "", + "正式库控制记录。", + "", + "- `recommend_analyze` 仅由 `has_pdf=true` 推导。", + "- `analyze` 控制是否生成正式文献卡片。", + "- `do_ocr` 控制 OCR 任务。", + "- `deep_reading_status` 仅两级:`pending`(未精读)/ `done`(已精读)。", + "", + ] + ) + return "\n".join(lines) + def _add_missing_frontmatter_fields(existing_content: str, new_fields: dict[str, str]) -> str: """Surgically append missing fields to existing frontmatter without overwriting anything.""" - if not existing_content.startswith('---'): + if not existing_content.startswith("---"): return existing_content - parts = existing_content.split('---', 2) + parts = existing_content.split("---", 2) if len(parts) < 3: return existing_content frontmatter = parts[1] body = parts[2] lines_to_add = [] for key, value in new_fields.items(): - pattern = '^' + re.escape(key) + '\\s*:' + pattern = "^" + re.escape(key) + "\\s*:" if not re.search(pattern, frontmatter, re.MULTILINE): - lines_to_add.append(f'{key}: {yaml_quote(value)}') + lines_to_add.append(f"{key}: {yaml_quote(value)}") if not lines_to_add: return existing_content - new_frontmatter = frontmatter.rstrip('\n') + '\n' + '\n'.join(lines_to_add) + '\n' - return f'---{new_frontmatter}---{body}' + new_frontmatter = frontmatter.rstrip("\n") + "\n" + "\n".join(lines_to_add) + "\n" + return f"---{new_frontmatter}---{body}" + def update_frontmatter_field(content: str, key: str, value: str) -> str: """Update an existing frontmatter field value, or add if missing.""" - if not content.startswith('---'): + if not content.startswith("---"): return content - pattern = '^' + re.escape(key) + '\\s*:.*$' - replacement = f'{key}: {yaml_quote(value)}' + pattern = "^" + re.escape(key) + "\\s*:.*$" + replacement = f"{key}: {yaml_quote(value)}" new_content, count = re.subn(pattern, replacement, content, flags=re.MULTILINE, count=1) if count == 0: new_content = _add_missing_frontmatter_fields(content, {key: value}) return new_content + def parse_existing_library_record(path: Path) -> dict: if not path.exists(): return {} - text = path.read_text(encoding='utf-8') + text = path.read_text(encoding="utf-8") result = {} - for key in ('analyze', 'recommend_analyze', 'do_ocr'): - match = re.search(f'^{key}:\\s*(true|false)$', text, re.MULTILINE) + for key in ("analyze", "recommend_analyze", "do_ocr"): + match = re.search(f"^{key}:\\s*(true|false)$", text, re.MULTILINE) if match: - result[key] = match.group(1) == 'true' - for key in ('ocr_status', 'analysis_note'): + result[key] = match.group(1) == "true" + for key in ("ocr_status", "analysis_note"): match = re.search(f'^{key}:\\s*"?(.*?)"?$', text, re.MULTILINE) if match: result[key] = match.group(1) - for key in ('deep_reading_status',): + for key in ("deep_reading_status",): match = re.search(f'^{key}:\\s*"?(.*?)"?$', text, re.MULTILINE) if match: result[key] = match.group(1) return result + def load_control_actions(paths: dict[str, Path]) -> dict[str, dict]: actions = {} - if not paths['library_records'].exists(): + if not paths["library_records"].exists(): return actions - for record in paths['library_records'].rglob('*.md'): - text = record.read_text(encoding='utf-8') - key_match = re.search('^zotero_key:\\s*(.+)$', text, re.MULTILINE) + for record in paths["library_records"].rglob("*.md"): + text = record.read_text(encoding="utf-8") + key_match = re.search("^zotero_key:\\s*(.+)$", text, re.MULTILINE) if not key_match: continue zotero_key = key_match.group(1).strip() row = parse_existing_library_record(record) - actions[zotero_key] = {'analyze': row.get('analyze', False), 'do_ocr': row.get('do_ocr', False)} + actions[zotero_key] = {"analyze": row.get("analyze", False), "do_ocr": row.get("do_ocr", False)} return actions + def run_selection_sync(vault: Path, verbose: bool = False) -> int: from paperforge.worker.base_views import ensure_base_views from paperforge.worker.ocr import validate_ocr_meta + paths = pipeline_paths(vault) config = load_domain_config(paths) ensure_base_views(vault, paths, config) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} + domain_lookup = {entry["export_file"]: entry["domain"] for entry in config["domains"]} written = 0 updated = 0 - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): domain = domain_lookup.get(export_path.name, export_path.stem) for item in load_export_rows(export_path): - pdf_attachments = [a for a in item.get('attachments', []) if a.get('contentType') == 'application/pdf'] + pdf_attachments = [a for a in item.get("attachments", []) if a.get("contentType") == "application/pdf"] has_pdf = bool(pdf_attachments) - raw_pdf_path = pdf_attachments[0].get('path', '') if pdf_attachments else '' + raw_pdf_path = pdf_attachments[0].get("path", "") if pdf_attachments else "" from paperforge.pdf_resolver import resolve_pdf_path - from paperforge.config import load_vault_config as _load_vault_config - cfg = _load_vault_config(vault) - zotero_dir = vault / cfg.get('system_dir', '99_System') / 'Zotero' + + cfg = load_vault_config(vault) + zotero_dir = vault / cfg.get("system_dir", "99_System") / "Zotero" resolved_pdf = resolve_pdf_path(raw_pdf_path, has_pdf, vault, zotero_dir) - collection_meta = collection_fields(item.get('collections', [])) - record_dir = paths['library_records'] / domain + collection_meta = collection_fields(item.get("collections", [])) + record_dir = paths["library_records"] / domain record_dir.mkdir(parents=True, exist_ok=True) record_path = record_dir / f"{item['key']}.md" existing = parse_existing_library_record(record_path) - meta_path = paths['ocr'] / item['key'] / 'meta.json' + meta_path = paths["ocr"] / item["key"] / "meta.json" meta = read_json(meta_path) if meta_path.exists() else {} - validated_ocr_status, validated_error = validate_ocr_meta(paths, meta) if meta else ('pending', '') + validated_ocr_status, validated_error = validate_ocr_meta(paths, meta) if meta else ("pending", "") if meta: - meta['ocr_status'] = validated_ocr_status + meta["ocr_status"] = validated_ocr_status if validated_error: - meta['error'] = validated_error + meta["error"] = validated_error write_json(meta_path, meta) - note_path = paths['literature'] / domain / f"{item['key']} - {slugify_filename(item['title'])}.md" - note_text = note_path.read_text(encoding='utf-8') if note_path.exists() else '' - fulltext_md_path = obsidian_wikilink_for_path(vault, meta.get('fulltext_md_path', '') or meta.get('markdown_path', '')) - ocr_status = meta.get('ocr_status', 'pending') - if not has_pdf or not resolved_pdf: - record_ocr_status = 'nopdf' - else: - record_ocr_status = ocr_status - creators = item.get('creators', []) - first_author = '' + note_path = paths["literature"] / domain / f"{item['key']} - {slugify_filename(item['title'])}.md" + note_text = note_path.read_text(encoding="utf-8") if note_path.exists() else "" + fulltext_md_path = obsidian_wikilink_for_path( + vault, meta.get("fulltext_md_path", "") or meta.get("markdown_path", "") + ) + ocr_status = meta.get("ocr_status", "pending") + record_ocr_status = "nopdf" if not has_pdf or not resolved_pdf else ocr_status + creators = item.get("creators", []) + first_author = "" for c in creators: - if c.get('creatorType') == 'author': + if c.get("creatorType") == "author": first_author = f"{c.get('firstName', '')} {c.get('lastName', '')}".strip() break - journal = item.get('publicationTitle', '') - extra = item.get('extra', '') + journal = item.get("publicationTitle", "") + extra = item.get("extra", "") impact_factor = lookup_impact_factor(journal, extra, vault) # Convert supplementary storage: paths to wikilinks supplementary_wikilinks = [] - for supp_path in item.get('supplementary', []): + for supp_path in item.get("supplementary", []): if supp_path: wikilink = obsidian_wikilink_for_pdf(supp_path, vault, zotero_dir) if wikilink: supplementary_wikilinks.append(wikilink) - pdf_wikilink = obsidian_wikilink_for_pdf(resolved_pdf, vault, zotero_dir) if resolved_pdf else '' - content = library_record_markdown({'zotero_key': item['key'], 'domain': domain, 'title': item.get('title', ''), 'year': item.get('year', ''), 'doi': item.get('doi', ''), 'date': item.get('date', ''), 'collection_path': ' | '.join(item.get('collections', [])), 'collections': collection_meta.get('collections', []), 'collection_tags': collection_meta.get('collection_tags', []), 'collection_group': collection_meta.get('collection_group', []), 'has_pdf': has_pdf, 'pdf_path': pdf_wikilink, 'bbt_path_raw': item.get('bbt_path_raw', ''), 'zotero_storage_key': item.get('zotero_storage_key', ''), 'attachment_count': item.get('attachment_count', 0), 'supplementary': supplementary_wikilinks, 'path_error': item.get('path_error', ''), 'recommend_analyze': bool(pdf_attachments), 'analyze': existing.get('analyze', False), 'do_ocr': existing.get('do_ocr', False), 'ocr_status': record_ocr_status, 'fulltext_md_path': fulltext_md_path, 'deep_reading_status': 'done' if note_text and has_deep_reading_content(note_text) else 'pending', 'analysis_note': existing.get('analysis_note', ''), 'first_author': first_author, 'journal': journal, 'impact_factor': impact_factor}) + pdf_wikilink = obsidian_wikilink_for_pdf(resolved_pdf, vault, zotero_dir) if resolved_pdf else "" + content = library_record_markdown( + { + "zotero_key": item["key"], + "domain": domain, + "title": item.get("title", ""), + "year": item.get("year", ""), + "doi": item.get("doi", ""), + "date": item.get("date", ""), + "collection_path": " | ".join(item.get("collections", [])), + "collections": collection_meta.get("collections", []), + "collection_tags": collection_meta.get("collection_tags", []), + "collection_group": collection_meta.get("collection_group", []), + "has_pdf": has_pdf, + "pdf_path": pdf_wikilink, + "bbt_path_raw": item.get("bbt_path_raw", ""), + "zotero_storage_key": item.get("zotero_storage_key", ""), + "attachment_count": item.get("attachment_count", 0), + "supplementary": supplementary_wikilinks, + "path_error": item.get("path_error", ""), + "recommend_analyze": bool(pdf_attachments), + "analyze": existing.get("analyze", False), + "do_ocr": existing.get("do_ocr", False), + "ocr_status": record_ocr_status, + "fulltext_md_path": fulltext_md_path, + "deep_reading_status": "done" if note_text and has_deep_reading_content(note_text) else "pending", + "analysis_note": existing.get("analysis_note", ""), + "first_author": first_author, + "journal": journal, + "impact_factor": impact_factor, + } + ) if record_path.exists(): - existing_content = record_path.read_text(encoding='utf-8') - updated_content = _add_missing_frontmatter_fields(existing_content, {'first_author': first_author, 'journal': journal, 'impact_factor': impact_factor, 'bbt_path_raw': item.get('bbt_path_raw', ''), 'zotero_storage_key': item.get('zotero_storage_key', ''), 'attachment_count': str(item.get('attachment_count', 0)), 'path_error': item.get('path_error', '')}) - updated_content = update_frontmatter_field(updated_content, 'has_pdf', has_pdf) - updated_content = update_frontmatter_field(updated_content, 'pdf_path', pdf_wikilink) - updated_content = update_frontmatter_field(updated_content, 'ocr_status', record_ocr_status) - updated_content = update_frontmatter_field(updated_content, 'deep_reading_status', 'done' if note_text and has_deep_reading_content(note_text) else 'pending') - updated_content = update_frontmatter_field(updated_content, 'fulltext_md_path', fulltext_md_path or '') + existing_content = record_path.read_text(encoding="utf-8") + updated_content = _add_missing_frontmatter_fields( + existing_content, + { + "first_author": first_author, + "journal": journal, + "impact_factor": impact_factor, + "bbt_path_raw": item.get("bbt_path_raw", ""), + "zotero_storage_key": item.get("zotero_storage_key", ""), + "attachment_count": str(item.get("attachment_count", 0)), + "path_error": item.get("path_error", ""), + }, + ) + updated_content = update_frontmatter_field(updated_content, "has_pdf", has_pdf) + updated_content = update_frontmatter_field(updated_content, "pdf_path", pdf_wikilink) + updated_content = update_frontmatter_field(updated_content, "ocr_status", record_ocr_status) + updated_content = update_frontmatter_field( + updated_content, + "deep_reading_status", + "done" if note_text and has_deep_reading_content(note_text) else "pending", + ) + updated_content = update_frontmatter_field(updated_content, "fulltext_md_path", fulltext_md_path or "") if updated_content != existing_content: - record_path.write_text(updated_content, encoding='utf-8') + record_path.write_text(updated_content, encoding="utf-8") updated += 1 else: written += 1 - record_path.write_text(content, encoding='utf-8') - print(f'selection-sync: wrote {written} records, updated {updated} records') + record_path.write_text(content, encoding="utf-8") + print(f"selection-sync: wrote {written} records, updated {updated} records") return 0 + def load_candidates_by_id(paths: dict[str, Path]) -> dict[str, dict]: - candidates = read_json(paths['candidates']) - return {row['candidate_id']: row for row in candidates} + candidates = read_json(paths["candidates"]) + return {row["candidate_id"]: row for row in candidates} + def save_candidates(paths: dict[str, Path], candidate_map: dict[str, dict]) -> None: collection_catalog = load_domain_collection_catalog(paths) @@ -894,171 +974,242 @@ def save_candidates(paths: dict[str, Path], candidate_map: dict[str, dict]) -> N rows = [] for row in candidate_map.values(): copy = dict(row) - copy['decision'] = canonicalize_decision(copy.get('decision', '')) + copy["decision"] = canonicalize_decision(copy.get("decision", "")) copy = apply_candidate_collection_resolution(copy, collection_catalog) copy = apply_existing_library_match(copy, export_inventory) - copy['final_collection'] = compute_final_collection(copy) + copy["final_collection"] = compute_final_collection(copy) rows.append(copy) - write_json(paths['candidates'], rows) + write_json(paths["candidates"], rows) + def writeback_command_for_candidate(row: dict) -> dict | None: - final_collection = str(row.get('final_collection', '') or '').strip() + final_collection = str(row.get("final_collection", "") or "").strip() if not final_collection: return None - candidate_id = str(row.get('candidate_id', '') or '').strip() + candidate_id = str(row.get("candidate_id", "") or "").strip() if not candidate_id: return None - command = {'command_id': f'wb-native-{candidate_id}', 'status': 'queued', 'source_candidate_id': candidate_id, 'target_domain': str(row.get('domain', '') or '').strip(), 'target_collection': final_collection, 'requested_at': datetime.now(timezone.utc).isoformat()} - existing_zotero_key = str(row.get('existing_zotero_key', '') or '').strip() + command = { + "command_id": f"wb-native-{candidate_id}", + "status": "queued", + "source_candidate_id": candidate_id, + "target_domain": str(row.get("domain", "") or "").strip(), + "target_collection": final_collection, + "requested_at": datetime.now(timezone.utc).isoformat(), + } + existing_zotero_key = str(row.get("existing_zotero_key", "") or "").strip() if existing_zotero_key: - command.update({'action': 'attach_existing_item_to_collection', 'existing_zotero_key': existing_zotero_key}) + command.update({"action": "attach_existing_item_to_collection", "existing_zotero_key": existing_zotero_key}) return command - doi = str(row.get('doi', '') or '').strip() - pmid = str(row.get('pmid', '') or '').strip() + doi = str(row.get("doi", "") or "").strip() + pmid = str(row.get("pmid", "") or "").strip() if doi: - command.update({'action': 'create_item_from_identifier', 'identifier_type': 'doi', 'identifier': doi, 'metadata_fallback': {'title': row.get('title', ''), 'authors': row.get('authors', []), 'year': str(row.get('year', '') or ''), 'journal': row.get('journal', ''), 'doi': doi, 'pmid': pmid, 'abstractNote': row.get('abstract_short', '')}}) + command.update( + { + "action": "create_item_from_identifier", + "identifier_type": "doi", + "identifier": doi, + "metadata_fallback": { + "title": row.get("title", ""), + "authors": row.get("authors", []), + "year": str(row.get("year", "") or ""), + "journal": row.get("journal", ""), + "doi": doi, + "pmid": pmid, + "abstractNote": row.get("abstract_short", ""), + }, + } + ) return command if pmid: - command.update({'action': 'create_item_from_identifier', 'identifier_type': 'pmid', 'identifier': pmid, 'metadata_fallback': {'title': row.get('title', ''), 'authors': row.get('authors', []), 'year': str(row.get('year', '') or ''), 'journal': row.get('journal', ''), 'doi': doi, 'pmid': pmid, 'abstractNote': row.get('abstract_short', '')}}) + command.update( + { + "action": "create_item_from_identifier", + "identifier_type": "pmid", + "identifier": pmid, + "metadata_fallback": { + "title": row.get("title", ""), + "authors": row.get("authors", []), + "year": str(row.get("year", "") or ""), + "journal": row.get("journal", ""), + "doi": doi, + "pmid": pmid, + "abstractNote": row.get("abstract_short", ""), + }, + } + ) return command - command.update({'action': 'create_item_from_metadata', 'metadata': {'title': row.get('title', ''), 'authors': row.get('authors', []), 'year': str(row.get('year', '') or ''), 'journal': row.get('journal', ''), 'doi': doi, 'pmid': pmid, 'abstractNote': row.get('abstract_short', '')}}) + command.update( + { + "action": "create_item_from_metadata", + "metadata": { + "title": row.get("title", ""), + "authors": row.get("authors", []), + "year": str(row.get("year", "") or ""), + "journal": row.get("journal", ""), + "doi": doi, + "pmid": pmid, + "abstractNote": row.get("abstract_short", ""), + }, + } + ) return command + def sync_writeback_queue(paths: dict[str, Path], candidate_map: dict[str, dict]) -> tuple[list[dict], int]: - existing_rows = read_jsonl(paths['queue']) - existing_by_candidate = {str(row.get('source_candidate_id', '') or '').strip(): dict(row) for row in existing_rows if str(row.get('source_candidate_id', '') or '').strip()} + existing_rows = read_jsonl(paths["queue"]) + existing_by_candidate = { + str(row.get("source_candidate_id", "") or "").strip(): dict(row) + for row in existing_rows + if str(row.get("source_candidate_id", "") or "").strip() + } queue_rows: list[dict] = [] queued_candidates: set[str] = set() created = 0 for candidate_id, row in candidate_map.items(): - decision = canonicalize_decision(row.get('decision', '')) - if decision != '纳入': + decision = canonicalize_decision(row.get("decision", "")) + if decision != "纳入": continue - if str(row.get('import_status', '') or '').strip() == 'imported': + if str(row.get("import_status", "") or "").strip() == "imported": continue candidate_copy = dict(row) - candidate_copy['final_collection'] = compute_final_collection(candidate_copy) - final_collection = str(candidate_copy.get('final_collection', '') or '').strip() + candidate_copy["final_collection"] = compute_final_collection(candidate_copy) + final_collection = str(candidate_copy.get("final_collection", "") or "").strip() if not final_collection: - candidate_copy['import_status'] = 'needs_collection_resolution' + candidate_copy["import_status"] = "needs_collection_resolution" candidate_map[candidate_id] = candidate_copy continue command = writeback_command_for_candidate(candidate_copy) if not command: - candidate_copy['import_status'] = 'blocked' + candidate_copy["import_status"] = "blocked" candidate_map[candidate_id] = candidate_copy continue existing = existing_by_candidate.get(candidate_id) - if existing and str(existing.get('status', '') or '').strip() in {'queued', 'running', 'processed'}: + if existing and str(existing.get("status", "") or "").strip() in {"queued", "running", "processed"}: merged = dict(existing) - merged['target_collection'] = command['target_collection'] - merged['target_domain'] = command.get('target_domain', merged.get('target_domain', '')) - if merged.get('status') != 'processed': - merged['requested_at'] = command['requested_at'] + merged["target_collection"] = command["target_collection"] + merged["target_domain"] = command.get("target_domain", merged.get("target_domain", "")) + if merged.get("status") != "processed": + merged["requested_at"] = command["requested_at"] queue_rows.append(merged) else: queue_rows.append(command) created += 1 - candidate_copy['import_status'] = 'queued_for_writeback' + candidate_copy["import_status"] = "queued_for_writeback" candidate_map[candidate_id] = candidate_copy queued_candidates.add(candidate_id) for row in existing_rows: - candidate_id = str(row.get('source_candidate_id', '') or '').strip() - status = str(row.get('status', '') or '').strip() + candidate_id = str(row.get("source_candidate_id", "") or "").strip() + status = str(row.get("status", "") or "").strip() if candidate_id in queued_candidates: continue - if status == 'processed': + if status == "processed": queue_rows.append(row) - write_jsonl(paths['queue'], queue_rows) + write_jsonl(paths["queue"], queue_rows) return (queue_rows, created) + def load_bridge_config(paths: dict[str, Path]) -> dict: - config_path = paths['bridge_config'] + config_path = paths["bridge_config"] if not config_path.exists(): - sample = read_json(paths['bridge_config_sample']) + sample = read_json(paths["bridge_config_sample"]) write_json(config_path, sample) return read_json(config_path) + def apply_writeback_log(paths: dict[str, Path], candidate_map: dict[str, dict]) -> int: - log_rows = read_jsonl(paths['log']) + log_rows = read_jsonl(paths["log"]) changed = 0 latest_by_candidate: dict[str, dict] = {} for row in log_rows: - candidate_id = str(row.get('source_candidate_id', '') or '').strip() + candidate_id = str(row.get("source_candidate_id", "") or "").strip() if candidate_id: latest_by_candidate[candidate_id] = row for candidate_id, log_row in latest_by_candidate.items(): candidate = dict(candidate_map.get(candidate_id, {})) if not candidate: continue - status = str(log_row.get('status', '') or '').strip() - if status == 'success': - candidate['import_status'] = 'imported' - candidate['zotero_key'] = str(log_row.get('zotero_key', '') or '').strip() + status = str(log_row.get("status", "") or "").strip() + if status == "success": + candidate["import_status"] = "imported" + candidate["zotero_key"] = str(log_row.get("zotero_key", "") or "").strip() changed += 1 - elif status == 'error': - candidate['import_status'] = 'writeback_error' + elif status == "error": + candidate["import_status"] = "writeback_error" changed += 1 candidate_map[candidate_id] = candidate return changed -def invoke_native_bridge(paths: dict[str, Path], max_commands: int=5) -> dict: + +def invoke_native_bridge(paths: dict[str, Path], max_commands: int = 5) -> dict: config = load_bridge_config(paths) - base_url = str(config.get('server_base_url', 'http://127.0.0.1:23119')).rstrip('/') - endpoint = str(config.get('process_endpoint', '/literaturePipeline/processQueue')) - payload = {'queuePath': str(paths['queue']).replace('\\', '/'), 'logPath': str(paths['log']).replace('\\', '/'), 'configPath': str(paths['bridge_config']).replace('\\', '/'), 'maxCommands': max_commands} - response = requests.post(f'{base_url}{endpoint}', json=payload, timeout=30) + base_url = str(config.get("server_base_url", "http://127.0.0.1:23119")).rstrip("/") + endpoint = str(config.get("process_endpoint", "/literaturePipeline/processQueue")) + payload = { + "queuePath": str(paths["queue"]).replace("\\", "/"), + "logPath": str(paths["log"]).replace("\\", "/"), + "configPath": str(paths["bridge_config"]).replace("\\", "/"), + "maxCommands": max_commands, + } + response = requests.post(f"{base_url}{endpoint}", json=payload, timeout=30) response.raise_for_status() result = response.json() - if not result.get('ok', False): - raise RuntimeError(result.get('error', 'Native bridge returned non-ok result')) + if not result.get("ok", False): + raise RuntimeError(result.get("error", "Native bridge returned non-ok result")) return result + def normalize_candidate_title(text: str) -> str: - return re.sub('\\s+', ' ', str(text or '').strip().lower()) + return re.sub("\\s+", " ", str(text or "").strip().lower()) + def candidate_identity_keys(row: dict) -> dict[str, str]: - doi = str(row.get('doi', '') or '').strip().lower() - pmid = str(row.get('pmid', '') or '').strip() - title = normalize_candidate_title(row.get('title', '')) - return {'doi': doi, 'pmid': pmid, 'title': title} + doi = str(row.get("doi", "") or "").strip().lower() + pmid = str(row.get("pmid", "") or "").strip() + title = normalize_candidate_title(row.get("title", "")) + return {"doi": doi, "pmid": pmid, "title": title} + def candidate_id_from_payload(row: dict) -> str: - source = str(row.get('source', '') or 'candidate').strip().lower() - doi = re.sub('[^a-z0-9]+', '-', str(row.get('doi', '') or '').strip().lower()).strip('-') - pmid = re.sub('[^0-9]+', '', str(row.get('pmid', '') or '').strip()) - fallback = re.sub('[^a-z0-9]+', '-', normalize_candidate_title(row.get('title', ''))).strip('-')[:80] - suffix = doi or pmid or fallback or datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') - return f'{source}-{suffix}' + source = str(row.get("source", "") or "candidate").strip().lower() + doi = re.sub("[^a-z0-9]+", "-", str(row.get("doi", "") or "").strip().lower()).strip("-") + pmid = re.sub("[^0-9]+", "", str(row.get("pmid", "") or "").strip()) + fallback = re.sub("[^a-z0-9]+", "-", normalize_candidate_title(row.get("title", ""))).strip("-")[:80] + suffix = doi or pmid or fallback or datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S") + return f"{source}-{suffix}" + def _normalize_candidate_value(value): if value is None: - return '' + return "" if isinstance(value, list): return [str(item).strip() for item in value if str(item).strip()] return str(value).strip() + def _authors_from_pubmed(value) -> list[str] | str: if isinstance(value, list): return [str(item).strip() for item in value if str(item).strip()] - return str(value or '').strip() + return str(value or "").strip() + def _authors_from_openalex(value) -> list[str]: authors = [] if isinstance(value, list): for item in value: if isinstance(item, dict): - author = item.get('author') or {} - name = author.get('display_name') or item.get('display_name') or '' + author = item.get("author") or {} + name = author.get("display_name") or item.get("display_name") or "" if name: authors.append(str(name).strip()) elif str(item).strip(): authors.append(str(item).strip()) return authors + def _abstract_from_openalex(inverted_index) -> str: if not isinstance(inverted_index, dict): - return '' + return "" tokens: list[tuple[int, str]] = [] for word, positions in inverted_index.items(): if not isinstance(positions, list): @@ -1069,193 +1220,385 @@ def _abstract_from_openalex(inverted_index) -> str: except Exception: continue if not tokens: - return '' + return "" tokens.sort(key=lambda item: item[0]) - return ' '.join((word for _, word in tokens)) + return " ".join((word for _, word in tokens)) + def _authors_from_arxiv(value) -> list[str]: authors = [] if isinstance(value, list): for item in value: if isinstance(item, dict): - name = item.get('name', '') or item.get('author', '') + name = item.get("name", "") or item.get("author", "") if name: authors.append(str(name).strip()) elif str(item).strip(): authors.append(str(item).strip()) return authors + def _authors_from_scholar(value) -> list[str]: if isinstance(value, list): return [str(item).strip() for item in value if str(item).strip()] if isinstance(value, str): - parts = re.split('\\s*,\\s*|\\s+and\\s+', value) + parts = re.split("\\s*,\\s*|\\s+and\\s+", value) return [part.strip() for part in parts if part.strip()] return [] + def adapt_pubmed_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - adapted = {'candidate_id': row.get('candidate_id') or f"pubmed-{payload.get('pmid') or payload.get('PMID') or ''}", 'domain': row.get('domain', ''), 'title': payload.get('title', ''), 'authors': _authors_from_pubmed(payload.get('authors', [])), 'year': payload.get('year', ''), 'journal': payload.get('journal', ''), 'doi': payload.get('doi', ''), 'pmid': payload.get('pmid') or payload.get('PMID', ''), 'source': row.get('source', '') or 'pubmed_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': payload.get('abstract', '') or payload.get('abstract_short', ''), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} + payload = dict(row.get("payload") or {}) + adapted = { + "candidate_id": row.get("candidate_id") or f"pubmed-{payload.get('pmid') or payload.get('PMID') or ''}", + "domain": row.get("domain", ""), + "title": payload.get("title", ""), + "authors": _authors_from_pubmed(payload.get("authors", [])), + "year": payload.get("year", ""), + "journal": payload.get("journal", ""), + "doi": payload.get("doi", ""), + "pmid": payload.get("pmid") or payload.get("PMID", ""), + "source": row.get("source", "") or "pubmed_search", + "requester_skill": row.get("requester_skill", ""), + "request_context": row.get("request_context", ""), + "abstract_short": payload.get("abstract", "") or payload.get("abstract_short", ""), + "decision": row.get("decision", ""), + "recommended_collection": row.get("recommended_collection", ""), + "recommend_confidence": row.get("recommend_confidence", ""), + "recommend_reason": row.get("recommend_reason", ""), + "user_collection": row.get("user_collection", ""), + "final_collection": row.get("final_collection", ""), + "duplicate_hint": row.get("duplicate_hint", ""), + "import_status": row.get("import_status", ""), + "note": row.get("note", ""), + "candidate_source_type": row.get("candidate_source_type", "") or "external_search", + "source_context": row.get("source_context", ""), + "task_relevance_reason": row.get("task_relevance_reason", ""), + } return adapted + def adapt_openalex_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - primary_location = payload.get('primary_location') or {} - source_info = primary_location.get('source') or {} - openalex_id = str(payload.get('id', '') or '').rstrip('/').split('/')[-1] - adapted = {'candidate_id': row.get('candidate_id') or f'openalex-{openalex_id}', 'domain': row.get('domain', ''), 'title': payload.get('display_name', '') or payload.get('title', ''), 'authors': _authors_from_openalex(payload.get('authorships', [])), 'year': payload.get('publication_year', '') or payload.get('year', ''), 'journal': source_info.get('display_name', '') or payload.get('journal', ''), 'doi': str(payload.get('doi', '') or '').replace('https://doi.org/', ''), 'pmid': '', 'source': row.get('source', '') or 'openalex_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': _abstract_from_openalex(payload.get('abstract_inverted_index')), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} + payload = dict(row.get("payload") or {}) + primary_location = payload.get("primary_location") or {} + source_info = primary_location.get("source") or {} + openalex_id = str(payload.get("id", "") or "").rstrip("/").split("/")[-1] + adapted = { + "candidate_id": row.get("candidate_id") or f"openalex-{openalex_id}", + "domain": row.get("domain", ""), + "title": payload.get("display_name", "") or payload.get("title", ""), + "authors": _authors_from_openalex(payload.get("authorships", [])), + "year": payload.get("publication_year", "") or payload.get("year", ""), + "journal": source_info.get("display_name", "") or payload.get("journal", ""), + "doi": str(payload.get("doi", "") or "").replace("https://doi.org/", ""), + "pmid": "", + "source": row.get("source", "") or "openalex_search", + "requester_skill": row.get("requester_skill", ""), + "request_context": row.get("request_context", ""), + "abstract_short": _abstract_from_openalex(payload.get("abstract_inverted_index")), + "decision": row.get("decision", ""), + "recommended_collection": row.get("recommended_collection", ""), + "recommend_confidence": row.get("recommend_confidence", ""), + "recommend_reason": row.get("recommend_reason", ""), + "user_collection": row.get("user_collection", ""), + "final_collection": row.get("final_collection", ""), + "duplicate_hint": row.get("duplicate_hint", ""), + "import_status": row.get("import_status", ""), + "note": row.get("note", ""), + "candidate_source_type": row.get("candidate_source_type", "") or "external_search", + "source_context": row.get("source_context", ""), + "task_relevance_reason": row.get("task_relevance_reason", ""), + } return adapted + def adapt_arxiv_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - arxiv_id = str(payload.get('id', '') or payload.get('entry_id', '')).rstrip('/').split('/')[-1] - adapted = {'candidate_id': row.get('candidate_id') or f'arxiv-{arxiv_id}', 'domain': row.get('domain', ''), 'title': payload.get('title', ''), 'authors': _authors_from_arxiv(payload.get('authors', [])), 'year': str(payload.get('published', '') or '')[:4], 'journal': payload.get('journal_ref', '') or 'arXiv', 'doi': payload.get('doi', ''), 'pmid': '', 'source': row.get('source', '') or 'arxiv_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': payload.get('summary', '') or payload.get('abstract', ''), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} + payload = dict(row.get("payload") or {}) + arxiv_id = str(payload.get("id", "") or payload.get("entry_id", "")).rstrip("/").split("/")[-1] + adapted = { + "candidate_id": row.get("candidate_id") or f"arxiv-{arxiv_id}", + "domain": row.get("domain", ""), + "title": payload.get("title", ""), + "authors": _authors_from_arxiv(payload.get("authors", [])), + "year": str(payload.get("published", "") or "")[:4], + "journal": payload.get("journal_ref", "") or "arXiv", + "doi": payload.get("doi", ""), + "pmid": "", + "source": row.get("source", "") or "arxiv_search", + "requester_skill": row.get("requester_skill", ""), + "request_context": row.get("request_context", ""), + "abstract_short": payload.get("summary", "") or payload.get("abstract", ""), + "decision": row.get("decision", ""), + "recommended_collection": row.get("recommended_collection", ""), + "recommend_confidence": row.get("recommend_confidence", ""), + "recommend_reason": row.get("recommend_reason", ""), + "user_collection": row.get("user_collection", ""), + "final_collection": row.get("final_collection", ""), + "duplicate_hint": row.get("duplicate_hint", ""), + "import_status": row.get("import_status", ""), + "note": row.get("note", ""), + "candidate_source_type": row.get("candidate_source_type", "") or "external_search", + "source_context": row.get("source_context", ""), + "task_relevance_reason": row.get("task_relevance_reason", ""), + } return adapted + def adapt_google_scholar_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - adapted = {'candidate_id': row.get('candidate_id') or f"google-scholar-{re.sub('[^a-z0-9]+', '-', normalize_candidate_title(payload.get('title', ''))).strip('-')[:80]}", 'domain': row.get('domain', ''), 'title': payload.get('title', ''), 'authors': _authors_from_scholar(payload.get('authors', [])), 'year': str(payload.get('year', '') or ''), 'journal': payload.get('journal', '') or payload.get('venue', ''), 'doi': payload.get('doi', ''), 'pmid': '', 'source': row.get('source', '') or 'google_scholar_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': payload.get('abstract', '') or payload.get('snippet', ''), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} + payload = dict(row.get("payload") or {}) + adapted = { + "candidate_id": row.get("candidate_id") + or f"google-scholar-{re.sub('[^a-z0-9]+', '-', normalize_candidate_title(payload.get('title', ''))).strip('-')[:80]}", + "domain": row.get("domain", ""), + "title": payload.get("title", ""), + "authors": _authors_from_scholar(payload.get("authors", [])), + "year": str(payload.get("year", "") or ""), + "journal": payload.get("journal", "") or payload.get("venue", ""), + "doi": payload.get("doi", ""), + "pmid": "", + "source": row.get("source", "") or "google_scholar_search", + "requester_skill": row.get("requester_skill", ""), + "request_context": row.get("request_context", ""), + "abstract_short": payload.get("abstract", "") or payload.get("snippet", ""), + "decision": row.get("decision", ""), + "recommended_collection": row.get("recommended_collection", ""), + "recommend_confidence": row.get("recommend_confidence", ""), + "recommend_reason": row.get("recommend_reason", ""), + "user_collection": row.get("user_collection", ""), + "final_collection": row.get("final_collection", ""), + "duplicate_hint": row.get("duplicate_hint", ""), + "import_status": row.get("import_status", ""), + "note": row.get("note", ""), + "candidate_source_type": row.get("candidate_source_type", "") or "external_search", + "source_context": row.get("source_context", ""), + "task_relevance_reason": row.get("task_relevance_reason", ""), + } return adapted + def adapt_candidate_event(row: dict) -> dict: - adapter = str(row.get('adapter', '') or '').strip() - if adapter == 'pubmed_search': + adapter = str(row.get("adapter", "") or "").strip() + if adapter == "pubmed_search": return adapt_pubmed_candidate(row) - if adapter == 'openalex_search': + if adapter == "openalex_search": return adapt_openalex_candidate(row) - if adapter == 'arxiv_search': + if adapter == "arxiv_search": return adapt_arxiv_candidate(row) - if adapter == 'google_scholar_search': + if adapter == "google_scholar_search": return adapt_google_scholar_candidate(row) return dict(row) + def default_user_agent() -> str: - return 'ResearchLiteraturePipeline/1.0 (+local-vault)' + return "ResearchLiteraturePipeline/1.0 (+local-vault)" + def build_search_event(base_task: dict, source: str, payload: dict) -> dict: - return {'adapter': source, 'payload': payload, 'source': source, 'requester_skill': base_task.get('requester_skill', ''), 'request_context': base_task.get('request_context', ''), 'domain': base_task.get('domain', ''), 'recommended_collection': base_task.get('recommended_collection', ''), 'recommend_confidence': base_task.get('recommend_confidence', ''), 'recommend_reason': base_task.get('recommend_reason', '') or f'{source} 检索命中', 'candidate_source_type': base_task.get('candidate_source_type', '') or 'external_search', 'task_relevance_reason': base_task.get('task_relevance_reason', ''), 'note': base_task.get('note', '')} + return { + "adapter": source, + "payload": payload, + "source": source, + "requester_skill": base_task.get("requester_skill", ""), + "request_context": base_task.get("request_context", ""), + "domain": base_task.get("domain", ""), + "recommended_collection": base_task.get("recommended_collection", ""), + "recommend_confidence": base_task.get("recommend_confidence", ""), + "recommend_reason": base_task.get("recommend_reason", "") or f"{source} 检索命中", + "candidate_source_type": base_task.get("candidate_source_type", "") or "external_search", + "task_relevance_reason": base_task.get("task_relevance_reason", ""), + "note": base_task.get("note", ""), + } + def _pubmed_abstract_and_doi_map(xml_text: str) -> dict[str, dict]: root = ET.fromstring(xml_text) result = {} - for article in root.findall('.//PubmedArticle'): - pmid = (article.findtext('.//MedlineCitation/PMID') or '').strip() + for article in root.findall(".//PubmedArticle"): + pmid = (article.findtext(".//MedlineCitation/PMID") or "").strip() if not pmid: continue abstract_parts = [] - for node in article.findall('.//Abstract/AbstractText'): - label = (node.attrib.get('Label') or '').strip() - text = ' '.join(''.join(node.itertext()).split()) + for node in article.findall(".//Abstract/AbstractText"): + label = (node.attrib.get("Label") or "").strip() + text = " ".join("".join(node.itertext()).split()) if not text: continue - abstract_parts.append(f'{label}: {text}' if label else text) - doi = '' - for id_node in article.findall('.//PubmedData/ArticleIdList/ArticleId'): - if (id_node.attrib.get('IdType') or '').lower() == 'doi': - doi = ''.join(id_node.itertext()).strip() + abstract_parts.append(f"{label}: {text}" if label else text) + doi = "" + for id_node in article.findall(".//PubmedData/ArticleIdList/ArticleId"): + if (id_node.attrib.get("IdType") or "").lower() == "doi": + doi = "".join(id_node.itertext()).strip() if doi: break if not doi: - for id_node in article.findall('.//ELocationID'): - if (id_node.attrib.get('EIdType') or '').lower() == 'doi': - doi = ''.join(id_node.itertext()).strip() + for id_node in article.findall(".//ELocationID"): + if (id_node.attrib.get("EIdType") or "").lower() == "doi": + doi = "".join(id_node.itertext()).strip() if doi: break - result[pmid] = {'abstract': '\n'.join(abstract_parts).strip(), 'doi': doi} + result[pmid] = {"abstract": "\n".join(abstract_parts).strip(), "doi": doi} return result + def search_pubmed(task: dict, limit: int) -> tuple[list[dict], dict]: - query = str(task.get('query', '') or '').strip() + query = str(task.get("query", "") or "").strip() if not query: - return ([], {'count': 0, 'ids': []}) - base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils' - headers = {'User-Agent': os.environ.get('LIT_PIPELINE_USER_AGENT', default_user_agent())} + return ([], {"count": 0, "ids": []}) + base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils" + headers = {"User-Agent": os.environ.get("LIT_PIPELINE_USER_AGENT", default_user_agent())} common_params = {} - email = os.environ.get('NCBI_EMAIL', '').strip() - api_key = os.environ.get('NCBI_API_KEY', '').strip() + email = os.environ.get("NCBI_EMAIL", "").strip() + api_key = os.environ.get("NCBI_API_KEY", "").strip() if email: - common_params['email'] = email + common_params["email"] = email if api_key: - common_params['api_key'] = api_key - esearch = requests.get(f'{base_url}/esearch.fcgi', params={'db': 'pubmed', 'retmode': 'json', 'sort': 'relevance', 'term': query, 'retmax': limit, **common_params}, headers=headers, timeout=60) + common_params["api_key"] = api_key + esearch = requests.get( + f"{base_url}/esearch.fcgi", + params={ + "db": "pubmed", + "retmode": "json", + "sort": "relevance", + "term": query, + "retmax": limit, + **common_params, + }, + headers=headers, + timeout=60, + ) esearch.raise_for_status() - search_payload = esearch.json().get('esearchresult', {}) - ids = [str(item).strip() for item in search_payload.get('idlist', []) if str(item).strip()] + search_payload = esearch.json().get("esearchresult", {}) + ids = [str(item).strip() for item in search_payload.get("idlist", []) if str(item).strip()] if not ids: - return ([], {'count': 0, 'ids': []}) - id_text = ','.join(ids) - esummary = requests.get(f'{base_url}/esummary.fcgi', params={'db': 'pubmed', 'retmode': 'json', 'id': id_text, **common_params}, headers=headers, timeout=60) + return ([], {"count": 0, "ids": []}) + id_text = ",".join(ids) + esummary = requests.get( + f"{base_url}/esummary.fcgi", + params={"db": "pubmed", "retmode": "json", "id": id_text, **common_params}, + headers=headers, + timeout=60, + ) esummary.raise_for_status() - summary_payload = esummary.json().get('result', {}) - efetch = requests.get(f'{base_url}/efetch.fcgi', params={'db': 'pubmed', 'retmode': 'xml', 'id': id_text, **common_params}, headers=headers, timeout=90) + summary_payload = esummary.json().get("result", {}) + efetch = requests.get( + f"{base_url}/efetch.fcgi", + params={"db": "pubmed", "retmode": "xml", "id": id_text, **common_params}, + headers=headers, + timeout=90, + ) efetch.raise_for_status() abstract_map = _pubmed_abstract_and_doi_map(efetch.text) rows = [] for pmid in ids: item = summary_payload.get(pmid, {}) or {} - title = html.unescape(str(item.get('title', '') or '')).strip() + title = html.unescape(str(item.get("title", "") or "")).strip() if not title: continue - article_ids = item.get('articleids', []) or [] - doi = '' + article_ids = item.get("articleids", []) or [] + doi = "" for article_id in article_ids: - if str(article_id.get('idtype', '')).lower() == 'doi': - doi = str(article_id.get('value', '')).strip() + if str(article_id.get("idtype", "")).lower() == "doi": + doi = str(article_id.get("value", "")).strip() if doi: break if not doi: - doi = abstract_map.get(pmid, {}).get('doi', '') - rows.append({'pmid': pmid, 'title': title, 'authors': [author.get('name', '') for author in item.get('authors') or [] if author.get('name')], 'year': _extract_year(str(item.get('pubdate', '') or '')), 'journal': item.get('fulljournalname', '') or item.get('source', ''), 'doi': doi, 'abstract': abstract_map.get(pmid, {}).get('abstract', '')}) - return (rows, {'count': len(rows), 'ids': ids}) + doi = abstract_map.get(pmid, {}).get("doi", "") + rows.append( + { + "pmid": pmid, + "title": title, + "authors": [author.get("name", "") for author in item.get("authors") or [] if author.get("name")], + "year": _extract_year(str(item.get("pubdate", "") or "")), + "journal": item.get("fulljournalname", "") or item.get("source", ""), + "doi": doi, + "abstract": abstract_map.get(pmid, {}).get("abstract", ""), + } + ) + return (rows, {"count": len(rows), "ids": ids}) + def search_openalex(task: dict, limit: int) -> tuple[list[dict], dict]: - query = str(task.get('query', '') or '').strip() + query = str(task.get("query", "") or "").strip() if not query: - return ([], {'count': 0}) - headers = {'User-Agent': os.environ.get('LIT_PIPELINE_USER_AGENT', default_user_agent())} - params = {'search': query, 'per-page': limit} - api_key = os.environ.get('OPENALEX_API_KEY', '').strip() + return ([], {"count": 0}) + headers = {"User-Agent": os.environ.get("LIT_PIPELINE_USER_AGENT", default_user_agent())} + params = {"search": query, "per-page": limit} + api_key = os.environ.get("OPENALEX_API_KEY", "").strip() if api_key: - params['api_key'] = api_key - mailto = os.environ.get('OPENALEX_MAILTO', '').strip() + params["api_key"] = api_key + mailto = os.environ.get("OPENALEX_MAILTO", "").strip() if mailto: - params['mailto'] = mailto - response = requests.get('https://api.openalex.org/works', params=params, headers=headers, timeout=60) + params["mailto"] = mailto + response = requests.get("https://api.openalex.org/works", params=params, headers=headers, timeout=60) response.raise_for_status() payload = response.json() - results = payload.get('results', []) or [] - return (results, {'count': len(results), 'meta': payload.get('meta', {})}) + results = payload.get("results", []) or [] + return (results, {"count": len(results), "meta": payload.get("meta", {})}) + def search_arxiv(task: dict, limit: int) -> tuple[list[dict], dict]: - query = str(task.get('query', '') or '').strip() + query = str(task.get("query", "") or "").strip() if not query: - return ([], {'count': 0}) - encoded_query = urllib.parse.quote(f'all:{query}') - url = f'https://export.arxiv.org/api/query?search_query={encoded_query}&start=0&max_results={limit}&sortBy=relevance&sortOrder=descending' - headers = {'User-Agent': os.environ.get('LIT_PIPELINE_USER_AGENT', default_user_agent())} + return ([], {"count": 0}) + encoded_query = urllib.parse.quote(f"all:{query}") + url = f"https://export.arxiv.org/api/query?search_query={encoded_query}&start=0&max_results={limit}&sortBy=relevance&sortOrder=descending" + headers = {"User-Agent": os.environ.get("LIT_PIPELINE_USER_AGENT", default_user_agent())} response = requests.get(url, headers=headers, timeout=60) response.raise_for_status() - ns = {'atom': 'http://www.w3.org/2005/Atom', 'arxiv': 'http://arxiv.org/schemas/atom'} + ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"} root = ET.fromstring(response.text) entries = [] - for entry in root.findall('atom:entry', ns): - entries.append({'id': (entry.findtext('atom:id', default='', namespaces=ns) or '').strip(), 'title': ' '.join((entry.findtext('atom:title', default='', namespaces=ns) or '').split()), 'summary': ' '.join((entry.findtext('atom:summary', default='', namespaces=ns) or '').split()), 'published': (entry.findtext('atom:published', default='', namespaces=ns) or '').strip(), 'authors': [{'name': (node.findtext('atom:name', default='', namespaces=ns) or '').strip()} for node in entry.findall('atom:author', ns)], 'doi': (entry.findtext('arxiv:doi', default='', namespaces=ns) or '').strip(), 'journal_ref': (entry.findtext('arxiv:journal_ref', default='', namespaces=ns) or '').strip()}) - return (entries, {'count': len(entries)}) + for entry in root.findall("atom:entry", ns): + entries.append( + { + "id": (entry.findtext("atom:id", default="", namespaces=ns) or "").strip(), + "title": " ".join((entry.findtext("atom:title", default="", namespaces=ns) or "").split()), + "summary": " ".join((entry.findtext("atom:summary", default="", namespaces=ns) or "").split()), + "published": (entry.findtext("atom:published", default="", namespaces=ns) or "").strip(), + "authors": [ + {"name": (node.findtext("atom:name", default="", namespaces=ns) or "").strip()} + for node in entry.findall("atom:author", ns) + ], + "doi": (entry.findtext("arxiv:doi", default="", namespaces=ns) or "").strip(), + "journal_ref": (entry.findtext("arxiv:journal_ref", default="", namespaces=ns) or "").strip(), + } + ) + return (entries, {"count": len(entries)}) + def _coerce_source_name(value: str) -> str: - text = str(value or '').strip().lower() - aliases = {'pubmed': 'pubmed_search', 'pubmed_search': 'pubmed_search', 'openalex': 'openalex_search', 'openalex_search': 'openalex_search', 'arxiv': 'arxiv_search', 'arxiv_search': 'arxiv_search'} + text = str(value or "").strip().lower() + aliases = { + "pubmed": "pubmed_search", + "pubmed_search": "pubmed_search", + "openalex": "openalex_search", + "openalex_search": "openalex_search", + "arxiv": "arxiv_search", + "arxiv_search": "arxiv_search", + } return aliases.get(text, text) + def run_search_command(vault: Path, args) -> int: paths = pipeline_paths(vault) - paths['search_tasks'].mkdir(parents=True, exist_ok=True) + paths["search_tasks"].mkdir(parents=True, exist_ok=True) task_id = f"search-{datetime.now().strftime('%Y%m%d-%H%M%S')}" - sources = args.sources or ['pubmed_search', 'openalex_search', 'arxiv_search'] - task = {'task_id': task_id, 'query': args.query, 'domain': args.domain, 'recommended_collection': args.recommended_collection or '', 'requester_skill': args.requester_skill or '', 'request_context': args.request_context or '', 'sources': sources, 'limit': args.limit, 'recommend_reason': args.recommend_reason or '围绕检索主题补充候选文献', 'candidate_source_type': 'external_search'} - task_path = paths['search_tasks'] / f'{task_id}.json' + sources = args.sources or ["pubmed_search", "openalex_search", "arxiv_search"] + task = { + "task_id": task_id, + "query": args.query, + "domain": args.domain, + "recommended_collection": args.recommended_collection or "", + "requester_skill": args.requester_skill or "", + "request_context": args.request_context or "", + "sources": sources, + "limit": args.limit, + "recommend_reason": args.recommend_reason or "围绕检索主题补充候选文献", + "candidate_source_type": "external_search", + } + task_path = paths["search_tasks"] / f"{task_id}.json" write_json(task_path, task) - print(f'search: task written -> {task_path}') + print(f"search: task written -> {task_path}") code = run_search_sources(vault) if code: return code @@ -1263,147 +1606,244 @@ def run_search_command(vault: Path, args) -> int: return run_ingest_candidates(vault) return 0 + def normalize_candidate_payload(row: dict) -> dict: normalized = {key: _normalize_candidate_value(value) for key, value in adapt_candidate_event(row).items()} - normalized['candidate_id'] = str(normalized.get('candidate_id', '') or '').strip() or candidate_id_from_payload(normalized) - normalized['title'] = str(normalized.get('title', '') or '').strip() - normalized['domain'] = str(normalized.get('domain', '') or '').strip() - normalized['source'] = str(normalized.get('source', '') or '').strip() or 'candidate_ingest' - normalized['candidate_source_type'] = str(normalized.get('candidate_source_type', '') or '').strip() or normalized['source'] - normalized['decision'] = canonicalize_decision(normalized.get('decision', '')) - normalized['import_status'] = str(normalized.get('import_status', '') or '').strip() or 'pending' - normalized['recommend_confidence'] = str(normalized.get('recommend_confidence', '') or '').strip() or '0' - normalized['status'] = 'candidate' + normalized["candidate_id"] = str(normalized.get("candidate_id", "") or "").strip() or candidate_id_from_payload( + normalized + ) + normalized["title"] = str(normalized.get("title", "") or "").strip() + normalized["domain"] = str(normalized.get("domain", "") or "").strip() + normalized["source"] = str(normalized.get("source", "") or "").strip() or "candidate_ingest" + normalized["candidate_source_type"] = ( + str(normalized.get("candidate_source_type", "") or "").strip() or normalized["source"] + ) + normalized["decision"] = canonicalize_decision(normalized.get("decision", "")) + normalized["import_status"] = str(normalized.get("import_status", "") or "").strip() or "pending" + normalized["recommend_confidence"] = str(normalized.get("recommend_confidence", "") or "").strip() or "0" + normalized["status"] = "candidate" return normalized + def merge_candidate_record(existing: dict | None, incoming: dict) -> dict: merged = dict(existing or {}) - preserve_if_existing = {'decision', 'user_collection', 'note', 'import_status'} + preserve_if_existing = {"decision", "user_collection", "note", "import_status"} for key, value in incoming.items(): if existing and key in preserve_if_existing: - current = merged.get(key, '') + current = merged.get(key, "") if str(current).strip(): continue merged[key] = value - merged['decision'] = canonicalize_decision(merged.get('decision', '')) - merged['final_collection'] = compute_final_collection(merged) + merged["decision"] = canonicalize_decision(merged.get("decision", "")) + merged["final_collection"] = compute_final_collection(merged) return merged + def resolve_existing_candidate(candidate_map: dict[str, dict], incoming: dict) -> tuple[str | None, dict | None]: - candidate_id = incoming.get('candidate_id', '') + candidate_id = incoming.get("candidate_id", "") if candidate_id and candidate_id in candidate_map: return (candidate_id, candidate_map[candidate_id]) incoming_keys = candidate_identity_keys(incoming) for existing_id, existing in candidate_map.items(): existing_keys = candidate_identity_keys(existing) - if incoming_keys['doi'] and incoming_keys['doi'] == existing_keys['doi']: + if incoming_keys["doi"] and incoming_keys["doi"] == existing_keys["doi"]: return (existing_id, existing) - if incoming_keys['pmid'] and incoming_keys['pmid'] == existing_keys['pmid']: + if incoming_keys["pmid"] and incoming_keys["pmid"] == existing_keys["pmid"]: return (existing_id, existing) - if incoming_keys['title'] and incoming_keys['title'] == existing_keys['title']: + if incoming_keys["title"] and incoming_keys["title"] == existing_keys["title"]: return (existing_id, existing) return (None, None) + def _harvest_csv_paths(paths: dict[str, Path]) -> list[Path]: - root = paths['harvest_root'] + root = paths["harvest_root"] if not root.exists(): return [] - return sorted(root.rglob('*-05-reference-harvest-candidates.csv')) + return sorted(root.rglob("*-05-reference-harvest-candidates.csv")) + def _normalize_harvest_value(value: str) -> str: - text = str(value or '').strip() - if text == '': - return '' + text = str(value or "").strip() + if text == "": + return "" return text + def _normalize_harvest_row(row: dict[str, str]) -> dict: normalized = normalize_candidate_payload({key: _normalize_harvest_value(value) for key, value in row.items()}) - normalized['source'] = normalized.get('source', '') or 'reference_harvest' - normalized['candidate_source_type'] = normalized.get('candidate_source_type', '') or 'reference_harvest' + normalized["source"] = normalized.get("source", "") or "reference_harvest" + normalized["candidate_source_type"] = normalized.get("candidate_source_type", "") or "reference_harvest" return normalized + def _merge_harvest_candidate(existing: dict | None, incoming: dict) -> dict: return merge_candidate_record(existing, incoming) + def next_key(domain: str, export_rows: list[dict]) -> str: - prefix = 'ORTHO' if domain == '骨科' else 'SPORT' - existing = [row.get('key', '') for row in export_rows] + prefix = "ORTHO" if domain == "骨科" else "SPORT" + existing = [row.get("key", "") for row in export_rows] max_num = 0 for key in existing: if key.startswith(prefix): - suffix = key[len(prefix):] + suffix = key[len(prefix) :] if suffix.isdigit(): max_num = max(max_num, int(suffix)) - return f'{prefix}{max_num + 1:03d}' + return f"{prefix}{max_num + 1:03d}" -def frontmatter_note(entry: dict, existing_text: str='') -> str: - deep_reading_path = entry.get('deep_reading_md_path', '') + +def frontmatter_note(entry: dict, existing_text: str = "") -> str: + deep_reading_path = entry.get("deep_reading_md_path", "") preserved_deep = extract_preserved_deep_reading(existing_text) - lines = ['---', f"title: {yaml_quote(entry['title'])}", f"year: {entry.get('year', '')}", 'type: article', f"journal: {yaml_quote(entry.get('journal', ''))}", 'authors:'] - for author in entry.get('authors', []): - lines.append(f' - {yaml_quote(author)}') - lines.extend([f"collection_path: {yaml_quote(entry.get('collection_path', ''))}", f"domain: {yaml_quote(entry.get('domain', ''))}", f"zotero_key: {yaml_quote(entry.get('zotero_key', ''))}", f"doi: {yaml_quote(entry.get('doi', ''))}", f"pmid: {yaml_quote(entry.get('pmid', ''))}"]) - lines.extend(yaml_list('collection_group', entry.get('collection_group', []))) - lines.extend(yaml_list('collections', entry.get('collections', []))) - lines.extend(yaml_list('collection_tags', entry.get('collection_tags', []))) - lines.extend(yaml_block(entry.get('abstract', ''))) - lines.extend([f"has_pdf: {('true' if entry.get('has_pdf') else 'false')}", f"ocr_status: {yaml_quote(entry.get('ocr_status', 'pending'))}", f"ocr_job_id: {yaml_quote(entry.get('ocr_job_id', ''))}", f"ocr_md_path: {yaml_quote(entry.get('ocr_md_path', ''))}", f"ocr_json_path: {yaml_quote(entry.get('ocr_json_path', ''))}", f"deep_reading_status: {yaml_quote(entry.get('deep_reading_status', 'pending'))}", f'deep_reading_md_path: {yaml_quote(deep_reading_path)}', f"pdf_path: {yaml_quote(entry.get('pdf_path', ''))}", 'tags:', ' - 文献阅读', f" - {entry.get('domain', '')}", '---', '', f"# {entry['title']}", '', '## 📄 文献基本信息', '', f"- Zotero Key: `{entry.get('zotero_key', '')}`", f"- Collection: `{entry.get('collection_path', '')}`", f"- 作者:{', '.join(entry.get('authors', []))}", f"- PDF: {('已检测' if entry.get('has_pdf') else '未检测到')}", f"- OCR: {entry.get('ocr_status', 'pending')}", f"- 精读: {entry.get('deep_reading_status', 'pending')}", '', '## 摘要', '', entry.get('abstract', '') or '暂无摘要', '', '## 💡 文献内容总结', '', '- 由 sync worker 自动生成的正式文献卡片。', '- 精读笔记(Deep Reading)仅由 /pf-deep 命令维护;sync --index 只保留已有内容,不自动生成。', '- 如需精读,请在 Base 中勾选 analyze,OCR 完成后运行 /pf-deep 。', '']) + lines = [ + "---", + f"title: {yaml_quote(entry['title'])}", + f"year: {entry.get('year', '')}", + "type: article", + f"journal: {yaml_quote(entry.get('journal', ''))}", + "authors:", + ] + for author in entry.get("authors", []): + lines.append(f" - {yaml_quote(author)}") + lines.extend( + [ + f"collection_path: {yaml_quote(entry.get('collection_path', ''))}", + f"domain: {yaml_quote(entry.get('domain', ''))}", + f"zotero_key: {yaml_quote(entry.get('zotero_key', ''))}", + f"doi: {yaml_quote(entry.get('doi', ''))}", + f"pmid: {yaml_quote(entry.get('pmid', ''))}", + ] + ) + lines.extend(yaml_list("collection_group", entry.get("collection_group", []))) + lines.extend(yaml_list("collections", entry.get("collections", []))) + lines.extend(yaml_list("collection_tags", entry.get("collection_tags", []))) + lines.extend(yaml_block(entry.get("abstract", ""))) + lines.extend( + [ + f"has_pdf: {('true' if entry.get('has_pdf') else 'false')}", + f"ocr_status: {yaml_quote(entry.get('ocr_status', 'pending'))}", + f"ocr_job_id: {yaml_quote(entry.get('ocr_job_id', ''))}", + f"ocr_md_path: {yaml_quote(entry.get('ocr_md_path', ''))}", + f"ocr_json_path: {yaml_quote(entry.get('ocr_json_path', ''))}", + f"deep_reading_status: {yaml_quote(entry.get('deep_reading_status', 'pending'))}", + f"deep_reading_md_path: {yaml_quote(deep_reading_path)}", + f"pdf_path: {yaml_quote(entry.get('pdf_path', ''))}", + "tags:", + " - 文献阅读", + f" - {entry.get('domain', '')}", + "---", + "", + f"# {entry['title']}", + "", + "## 📄 文献基本信息", + "", + f"- Zotero Key: `{entry.get('zotero_key', '')}`", + f"- Collection: `{entry.get('collection_path', '')}`", + f"- 作者:{', '.join(entry.get('authors', []))}", + f"- PDF: {('已检测' if entry.get('has_pdf') else '未检测到')}", + f"- OCR: {entry.get('ocr_status', 'pending')}", + f"- 精读: {entry.get('deep_reading_status', 'pending')}", + "", + "## 摘要", + "", + entry.get("abstract", "") or "暂无摘要", + "", + "## 💡 文献内容总结", + "", + "- 由 sync worker 自动生成的正式文献卡片。", + "- 精读笔记(Deep Reading)仅由 /pf-deep 命令维护;sync --index 只保留已有内容,不自动生成。", + "- 如需精读,请在 Base 中勾选 analyze,OCR 完成后运行 /pf-deep 。", + "", + ] + ) if preserved_deep: - lines.extend(['', preserved_deep, '']) - return '\n'.join(lines) + lines.extend(["", preserved_deep, ""]) + return "\n".join(lines) + def analyze_selected_keys(paths: dict[str, Path]) -> set[str]: - return {key for key, row in load_control_actions(paths).items() if row.get('analyze')} + return {key for key, row in load_control_actions(paths).items() if row.get("analyze")} + def run_index_refresh(vault: Path, verbose: bool = False) -> int: from paperforge.worker.base_views import ensure_base_views from paperforge.worker.ocr import validate_ocr_meta + paths = pipeline_paths(vault) config = load_domain_config(paths) ensure_base_views(vault, paths, config) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} - from paperforge.config import load_vault_config as _load_vault_config - cfg = _load_vault_config(vault) - zotero_dir = vault / cfg.get('system_dir', '99_System') / 'Zotero' + domain_lookup = {entry["export_file"]: entry["domain"] for entry in config["domains"]} + + cfg = load_vault_config(vault) + zotero_dir = vault / cfg.get("system_dir", "99_System") / "Zotero" exports = {} - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): domain = domain_lookup.get(export_path.name, export_path.stem) export_rows = load_export_rows(export_path) - exports[domain] = {row['key']: row for row in export_rows} + exports[domain] = {row["key"]: row for row in export_rows} selected_keys = None index_rows = [] - lit_root = paths['literature'] - for export_path in sorted(paths['exports'].glob('*.json')): + lit_root = paths["literature"] + for export_path in sorted(paths["exports"].glob("*.json")): domain = domain_lookup.get(export_path.name, export_path.stem) export_rows = load_export_rows(export_path) for item in export_rows: - key = item['key'] + key = item["key"] if selected_keys is not None and key not in selected_keys: continue - collection_meta = collection_fields(item.get('collections', [])) - pdf_attachments = [a for a in item.get('attachments', []) if a.get('contentType') == 'application/pdf'] - meta_path = paths['ocr'] / key / 'meta.json' + collection_meta = collection_fields(item.get("collections", [])) + pdf_attachments = [a for a in item.get("attachments", []) if a.get("contentType") == "application/pdf"] + meta_path = paths["ocr"] / key / "meta.json" meta = read_json(meta_path) if meta_path.exists() else {} if meta: validated_ocr_status, validated_error = validate_ocr_meta(paths, meta) - meta['ocr_status'] = validated_ocr_status + meta["ocr_status"] = validated_ocr_status if validated_error: - meta['error'] = validated_error + meta["error"] = validated_error write_json(meta_path, meta) - title_slug = slugify_filename(item['title']) - note_path = lit_root / domain / f'{key} - {title_slug}.md' + title_slug = slugify_filename(item["title"]) + note_path = lit_root / domain / f"{key} - {title_slug}.md" if note_path.parent.exists(): - for stale_note in note_path.parent.glob(f'{key} - *.md'): + for stale_note in note_path.parent.glob(f"{key} - *.md"): if stale_note != note_path: stale_note.unlink() - entry = {'zotero_key': key, 'domain': domain, 'title': item['title'], 'authors': item.get('authors', []), 'abstract': item.get('abstract', ''), 'journal': item.get('journal', ''), 'year': item.get('year', ''), 'doi': item.get('doi', ''), 'pmid': item.get('pmid', ''), 'collection_path': ' | '.join(item.get('collections', [])), 'collections': collection_meta.get('collections', []), 'collection_tags': collection_meta.get('collection_tags', []), 'collection_group': collection_meta.get('collection_group', []), 'has_pdf': bool(pdf_attachments), 'pdf_path': obsidian_wikilink_for_pdf(pdf_attachments[0]['path'], vault, zotero_dir) if pdf_attachments else '', 'ocr_status': meta.get('ocr_status', 'pending'), 'ocr_job_id': meta.get('ocr_job_id', ''), 'ocr_md_path': obsidian_wikilink_for_path(vault, meta.get('markdown_path', '')), 'ocr_json_path': meta.get('json_path', ''), 'deep_reading_status': 'done' if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding='utf-8')) else 'pending', 'note_path': str(note_path.relative_to(vault)).replace('\\', '/'), 'deep_reading_md_path': str(note_path.relative_to(vault)).replace('\\', '/') if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding='utf-8')) else ''} + entry = { + "zotero_key": key, + "domain": domain, + "title": item["title"], + "authors": item.get("authors", []), + "abstract": item.get("abstract", ""), + "journal": item.get("journal", ""), + "year": item.get("year", ""), + "doi": item.get("doi", ""), + "pmid": item.get("pmid", ""), + "collection_path": " | ".join(item.get("collections", [])), + "collections": collection_meta.get("collections", []), + "collection_tags": collection_meta.get("collection_tags", []), + "collection_group": collection_meta.get("collection_group", []), + "has_pdf": bool(pdf_attachments), + "pdf_path": obsidian_wikilink_for_pdf(pdf_attachments[0]["path"], vault, zotero_dir) + if pdf_attachments + else "", + "ocr_status": meta.get("ocr_status", "pending"), + "ocr_job_id": meta.get("ocr_job_id", ""), + "ocr_md_path": obsidian_wikilink_for_path(vault, meta.get("markdown_path", "")), + "ocr_json_path": meta.get("json_path", ""), + "deep_reading_status": "done" + if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding="utf-8")) + else "pending", + "note_path": str(note_path.relative_to(vault)).replace("\\", "/"), + "deep_reading_md_path": str(note_path.relative_to(vault)).replace("\\", "/") + if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding="utf-8")) + else "", + } note_path.parent.mkdir(parents=True, exist_ok=True) - existing_text = note_path.read_text(encoding='utf-8') if note_path.exists() else '' - note_path.write_text(frontmatter_note(entry, existing_text), encoding='utf-8') + existing_text = note_path.read_text(encoding="utf-8") if note_path.exists() else "" + note_path.write_text(frontmatter_note(entry, existing_text), encoding="utf-8") index_rows.append(entry) - write_json(paths['index'], index_rows) - print(f'index-refresh: wrote {len(index_rows)} index rows') - control_records_dir = paths['library_records'] + write_json(paths["index"], index_rows) + print(f"index-refresh: wrote {len(index_rows)} index rows") + control_records_dir = paths["library_records"] if control_records_dir.exists(): for domain_dir in control_records_dir.iterdir(): if not domain_dir.is_dir(): @@ -1412,15 +1852,20 @@ def run_index_refresh(vault: Path, verbose: bool = False) -> int: domain_export_keys = set(exports.get(domain, {}).keys()) records_by_title = {} records_info = {} - for record_file in domain_dir.glob('*.md'): + for record_file in domain_dir.glob("*.md"): try: - content = record_file.read_text(encoding='utf-8') - title_match = re.search('^title:\\s*["\\\']?(.+)["\\\']?\\s*$', content, re.MULTILINE) - title = title_match.group(1) if title_match else '' - has_pdf = 'has_pdf: true' in content - normalized = re.sub('[^a-z0-9]', '', title.lower())[:20] + content = record_file.read_text(encoding="utf-8") + title_match = re.search("^title:\\s*[\"\\']?(.+)[\"\\']?\\s*$", content, re.MULTILINE) + title = title_match.group(1) if title_match else "" + has_pdf = "has_pdf: true" in content + normalized = re.sub("[^a-z0-9]", "", title.lower())[:20] key = record_file.stem - records_info[key] = {'file': record_file, 'title': title, 'has_pdf': has_pdf, 'normalized': normalized} + records_info[key] = { + "file": record_file, + "title": title, + "has_pdf": has_pdf, + "normalized": normalized, + } if normalized not in records_by_title: records_by_title[normalized] = [] records_by_title[normalized].append(key) @@ -1432,16 +1877,15 @@ def run_index_refresh(vault: Path, verbose: bool = False) -> int: keys_not_in_export = [k for k in keys if k not in domain_export_keys] if keys_in_export and keys_not_in_export: for k in keys_not_in_export: - if not records_info[k]['has_pdf']: + if not records_info[k]["has_pdf"]: to_delete.append(k) deleted_count = 0 for key in to_delete: try: - records_info[key]['file'].unlink() + records_info[key]["file"].unlink() deleted_count += 1 except Exception: pass if deleted_count > 0: - print(f'index-refresh: cleaned {deleted_count} orphaned records in {domain}') + print(f"index-refresh: cleaned {deleted_count} orphaned records in {domain}") return 0 - diff --git a/paperforge/worker/update.py b/paperforge/worker/update.py index 8d9dde50..9e3b5d55 100644 --- a/paperforge/worker/update.py +++ b/paperforge/worker/update.py @@ -1,147 +1,26 @@ from __future__ import annotations -import logging -import argparse -import csv + import hashlib -import html import json +import logging import os -import re import shutil import subprocess import sys import tempfile import urllib.parse -from json import JSONDecodeError import zipfile -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path -from xml.etree import ElementTree as ET -import requests -import fitz -from PIL import Image + +from paperforge.config import load_vault_config, paperforge_paths +from paperforge.worker._utils import ( + read_json, + write_json, +) logger = logging.getLogger(__name__) -STANDARD_VIEW_NAMES = frozenset([ - "控制面板", "推荐分析", "待 OCR", "OCR 完成", - "待深度阅读", "深度阅读完成", "正式卡片", "全记录" -]) - -def load_simple_env(env_path: Path) -> None: - if not env_path.exists(): - return - for raw_line in env_path.read_text(encoding='utf-8').splitlines(): - line = raw_line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - key, value = line.split('=', 1) - key = key.strip() - if not key or key in os.environ: - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}): - value = value[1:-1] - os.environ[key] = value - -def read_json(path: Path): - return json.loads(path.read_text(encoding='utf-8')) -_JOURNAL_DB: dict[str, dict] | None = None - -def load_journal_db(vault: Path) -> dict[str, dict]: - """Load zoterostyle.json journal database.""" - global _JOURNAL_DB - if _JOURNAL_DB is not None: - return _JOURNAL_DB - zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json' - if zoterostyle_path.exists(): - try: - _JOURNAL_DB = read_json(zoterostyle_path) - except (JSONDecodeError, Exception): - _JOURNAL_DB = {} - else: - _JOURNAL_DB = {} - return _JOURNAL_DB - -def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str: - """Lookup impact factor: prefer zoterostyle.json, fallback to extra field.""" - if not journal_name: - return '' - journal_db = load_journal_db(vault) - if journal_name in journal_db: - rank_data = journal_db[journal_name].get('rank', {}) - if isinstance(rank_data, dict): - sciif = rank_data.get('sciif', '') - if sciif: - return str(sciif) - if extra: - if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra) - if if_match: - return if_match.group(1) - return '' - -def write_json(path: Path, data) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') - -def read_jsonl(path: Path): - rows = [] - if not path.exists(): - return rows - for line in path.read_text(encoding='utf-8').splitlines(): - line = line.strip() - if line: - rows.append(json.loads(line)) - return rows - -def write_jsonl(path: Path, rows) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows)) - if text: - text += '\n' - path.write_text(text, encoding='utf-8') - -def yaml_quote(value: str) -> str: - if isinstance(value, bool): - return 'true' if value else 'false' - return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"' - -def yaml_block(value: str) -> list[str]: - value = (value or '').strip() - if not value: - return ['abstract: |-', ' '] - lines = ['abstract: |-'] - for line in value.splitlines(): - lines.append(f' {line}') - return lines - -def yaml_list(key: str, values) -> list[str]: - cleaned = [str(value).strip() for value in values or [] if str(value).strip()] - if not cleaned: - return [f'{key}: []'] - lines = [f'{key}:'] - for value in cleaned: - lines.append(f' - {yaml_quote(value)}') - return lines - -def slugify_filename(text: str) -> str: - cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip() - return cleaned[:120] or 'untitled' - -def _extract_year(value: str) -> str: - match = re.search('(19|20)\\d{2}', value or '') - return match.group(0) if match else '' - - -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - def pipeline_paths(vault: Path) -> dict[str, Path]: """Build complete PaperForge path inventory — delegates to shared resolver. @@ -149,14 +28,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: Returns paths from paperforge.config.paperforge_paths() plus worker-only keys. Preserves all legacy keys for existing callers. """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] + shared = paperforge_paths(vault) root = shared["paperforge"] control_root = shared["control"] @@ -183,17 +55,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]: "ocr_queue": root / "ocr" / "ocr-queue.json", } + def load_domain_config(paths: dict[str, Path]) -> dict: """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} + config_path = paths["config"] + config = read_json(config_path) if config_path.exists() else {"domains": []} domains = config.setdefault("domains", []) known_exports = {str(entry.get("export_file", "")) for entry in domains} changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): + for export_path in sorted(paths["exports"].glob("*.json")): if export_path.name in known_exports: continue domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) @@ -204,6 +74,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict: write_json(config_path, config) return config + def protected_paths(vault: Path) -> set[str]: cfg = load_vault_config(vault) pf = f"{cfg['system_dir']}/PaperForge" @@ -219,11 +90,12 @@ def protected_paths(vault: Path) -> set[str]: } - def _remote_version() -> str | None: try: api = f"https://api.github.com/repos/{GITHUB_REPO}/contents/paperforge.json" - req = urllib.request.Request(api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"}) + req = urllib.request.Request( + api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"} + ) with urllib.request.urlopen(req, timeout=10) as resp: data = json.loads(resp.read()) req2 = urllib.request.Request(data["download_url"], headers={"User-Agent": "PaperForge"}) @@ -260,7 +132,7 @@ def _do_backup(vault: Path, updates: list) -> Path | None: backup_dir = vault / f".backup_{datetime.now():%Y%m%d_%H%M%S}" backup_dir.mkdir(exist_ok=True) count = 0 - for src, dst, action in updates: + for _src, dst, action in updates: if action == "UPDATE" and dst.exists(): bp = backup_dir / dst.relative_to(vault) bp.parent.mkdir(parents=True, exist_ok=True) @@ -273,7 +145,7 @@ def _do_backup(vault: Path, updates: list) -> Path | None: def _apply_updates(vault: Path, updates: list) -> bool: try: - for src, dst, action in updates: + for src, dst, _action in updates: dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst) return True @@ -295,21 +167,22 @@ def _rollback(vault: Path, backup_dir: Path) -> None: def _detect_install_method() -> tuple[str, Path | None]: """Detect how paperforge is installed.""" import paperforge + pkg_dir = Path(paperforge.__file__).parent.resolve() - + # Check if installed in site-packages (pip install) if "site-packages" in str(pkg_dir) or "dist-packages" in str(pkg_dir): return ("pip", pkg_dir) - + # Check if in editable mode (pip install -e .) if pkg_dir.name == "paperforge" and (pkg_dir.parent / ".git").exists(): return ("pip-editable", pkg_dir.parent) - + # Check if vault has .git (git clone) vault = Path.cwd() if (vault / ".git").exists(): return ("git", vault) - + return ("unknown", None) @@ -321,8 +194,8 @@ def _update_via_pip(editable: bool = False) -> bool: else: cmd.append("--upgrade") cmd.append("paperforge") - - logger.info("执行: %s", ' '.join(cmd)) + + logger.info("执行: %s", " ".join(cmd)) r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8") if r.returncode != 0: logger.error("pip 更新失败: %s", r.stderr) @@ -373,7 +246,7 @@ def update_via_zip(vault: Path) -> bool: logger.info("所有文件已是最新") return True logger.info("发现 %d 个文件需要更新:", len(updates)) - for src, dst, action in updates: + for _src, dst, action in updates: logger.info(" [%s] %s", action, dst.relative_to(vault)) backup = _do_backup(vault, updates) if _apply_updates(vault, updates): @@ -401,12 +274,14 @@ def run_update(vault: Path) -> int: logger.info("PaperForge Lite 更新") logger.info("%s", "=" * 50) logger.info("本地版本: %s", local) - logger.info("远程版本: %s", remote or 'unknown') + logger.info("远程版本: %s", remote or "unknown") if not remote: logger.error("无法获取远程版本") return 1 try: - needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple(int(x) for x in local.split(".") if x.isdigit()) + needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple( + int(x) for x in local.split(".") if x.isdigit() + ) except ValueError: needs = remote != local if not needs: @@ -418,11 +293,11 @@ def run_update(vault: Path) -> int: if ans not in ("y", "yes"): logger.info("已取消") return 0 - + # Auto-detect installation method method, path = _detect_install_method() logger.info("安装方式: %s", method) - + if method == "pip": logger.info("通过 pip 更新...") success = _update_via_pip(editable=False) @@ -443,7 +318,7 @@ def run_update(vault: Path) -> int: else: logger.warning("未检测到标准安装方式,尝试 zip 下载...") success = _update_via_zip(vault) - + if success: logger.info("更新完成!请重启 Obsidian") return 0 if success else 1 @@ -452,4 +327,3 @@ def run_update(vault: Path) -> int: # ============================================================================= # Main # ============================================================================= - diff --git a/pyproject.toml b/pyproject.toml index 03a05709..b186a408 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ [project.optional-dependencies] test = [ "pytest>=7.4.0", + "ruff>=0.4.0", ] [project.scripts] @@ -54,10 +55,29 @@ addopts = "--ignore=tests/sandbox/00_TestVault/" [tool.ruff] target-version = "py310" line-length = 120 +exclude = ["paperforge/skills"] [tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "SIM"] ignore = ["B904"] # raise-within-try is intentional in many worker error paths +[tool.ruff.lint.per-file-ignores] +# Pre-existing: undefined names in update.py (GITHUB_REPO etc. defined below function defs) +"paperforge/worker/update.py" = ["F821"] +# Pre-existing: undefined names in sync.py (run_search_sources, run_ingest_candidates) +"paperforge/worker/sync.py" = ["F821", "B007", "E501"] +# Pre-existing: style suggestions not part of dead code removal +"paperforge/worker/ocr.py" = ["SIM102", "E501", "B905"] +"paperforge/worker/repair.py" = ["SIM102"] +"paperforge/ocr_diagnostics.py" = ["E501", "SIM105"] +# Pre-existing: importability check in status.py uses try/except imports +"paperforge/worker/status.py" = ["F401"] +# Pre-existing: test files use patterns incompatible with strict ruff rules +"tests/**" = ["E501", "E402", "SIM117", "SIM108", "F841", "B007", "SIM103", "SIM105", "SIM222", "B905"] +# Pre-existing: scripts with formatting not part of this cleanup +"scripts/consistency_audit.py" = ["SIM110", "SIM102"] +"scripts/welcome.py" = ["E501"] +"scripts/validate_setup.py" = ["E501"] + [tool.ruff.format] quote-style = "double" diff --git a/scripts/consistency_audit.py b/scripts/consistency_audit.py index 9c13835a..4fb80c0b 100644 --- a/scripts/consistency_audit.py +++ b/scripts/consistency_audit.py @@ -12,11 +12,10 @@ Exit code: 0 if all pass, 1 if any fail. from __future__ import annotations -import os import re import sys +from collections.abc import Iterable from pathlib import Path -from typing import Iterable # Root of the repository (parent of this script) REPO_ROOT = Path(__file__).parent.parent.resolve() @@ -77,10 +76,7 @@ COMMAND_DOC_REQUIRED_SECTIONS = [ def should_exclude_check1(path: Path) -> bool: """Check if a path should be excluded from Check 1.""" rel = path.relative_to(REPO_ROOT).as_posix() - for exclude in CHECK1_EXCLUDES: - if rel.startswith(exclude) or exclude in rel: - return True - return False + return any(rel.startswith(exclude) or exclude in rel for exclude in CHECK1_EXCLUDES) def find_files(pattern: str, root: Path = REPO_ROOT) -> Iterable[Path]: @@ -111,7 +107,7 @@ def check_old_commands() -> tuple[int, list[str]]: line_end = len(content) line = content[line_start:line_end] # Allow old commands in explicit migration sections of AGENTS.md - if rel == "AGENTS.md" and "命令迁移说明" in content[:match.start()]: + if rel == "AGENTS.md" and "命令迁移说明" in content[: match.start()]: # Check if this is in section 11 (migration section) section_start = content.find("## 11. 命令迁移说明") if section_start != -1 and match.start() > section_start: @@ -203,9 +199,7 @@ def check_dead_links() -> tuple[int, list[str]]: if not target_path.exists(): # Try appending .md for implicit markdown links if not (target_path.with_suffix(".md")).exists(): - violations.append( - f" [{rel}] Dead link to '{link_target}' (text: '{link_text}')" - ) + violations.append(f" [{rel}] Dead link to '{link_target}' (text: '{link_text}')") passed = len(violations) == 0 return (0 if passed else 1), violations @@ -243,6 +237,7 @@ def check_command_docs() -> tuple[int, list[str]]: def check_duplicate_utils() -> tuple[int, list[str]]: """Check 5: Detect worker modules with duplicated utility functions outside the re-export from _utils.py.""" import ast + violations: list[str] = [] worker_dir = REPO_ROOT / "paperforge" / "worker" @@ -250,13 +245,23 @@ def check_duplicate_utils() -> tuple[int, list[str]]: return 0, violations # Known shared function names from _utils.py - shared_funcs = frozenset({ - "read_json", "write_json", "read_jsonl", "write_jsonl", - "yaml_quote", "yaml_block", "yaml_list", - "slugify_filename", "_extract_year", - "load_journal_db", "lookup_impact_factor", - "scan_library_records", "_resolve_formal_note_path", - }) + shared_funcs = frozenset( + { + "read_json", + "write_json", + "read_jsonl", + "write_jsonl", + "yaml_quote", + "yaml_block", + "yaml_list", + "slugify_filename", + "_extract_year", + "load_journal_db", + "lookup_impact_factor", + "scan_library_records", + "_resolve_formal_note_path", + } + ) skip_files = frozenset({"_utils.py", "_retry.py", "_progress.py", "__init__.py"}) @@ -293,9 +298,7 @@ def check_duplicate_utils() -> tuple[int, list[str]]: if is_reexport: continue - violations.append( - f" [{rel}] Function '{node.name}' is defined locally but exists in _utils.py" - ) + violations.append(f" [{rel}] Function '{node.name}' is defined locally but exists in _utils.py") passed = len(violations) == 0 return (0 if passed else 1), violations diff --git a/tests/test_legacy_worker_compat.py b/tests/test_legacy_worker_compat.py index 3125e2d9..60d9736b 100644 --- a/tests/test_legacy_worker_compat.py +++ b/tests/test_legacy_worker_compat.py @@ -1,4 +1,5 @@ """Test compatibility: worker modules use shared resolver.""" + from __future__ import annotations import json @@ -22,7 +23,7 @@ def tmp_vault(tmp_path: Path) -> Path: (paperforge / "candidates").mkdir(parents=True) resources = tmp_path / "03_Resources" - literature = resources / "Literature" + resources / "Literature" control = resources / "LiteratureControl" (control / "library-records").mkdir(parents=True) @@ -58,13 +59,13 @@ class TestWorkerLoadVaultConfig: shared_cfg = shared_load(tmp_vault) worker_cfg = worker_load(tmp_vault) - assert set(worker_cfg.keys()) == set(shared_cfg.keys()), ( - f"Key mismatch: worker={set(worker_cfg.keys())} vs shared={set(shared_cfg.keys())}" - ) + assert set(worker_cfg.keys()) == set( + shared_cfg.keys() + ), f"Key mismatch: worker={set(worker_cfg.keys())} vs shared={set(shared_cfg.keys())}" for key in shared_cfg: - assert worker_cfg.get(key) == shared_cfg.get(key), ( - f"Key '{key}' differs: worker={worker_cfg.get(key)!r} vs shared={shared_cfg.get(key)!r}" - ) + assert worker_cfg.get(key) == shared_cfg.get( + key + ), f"Key '{key}' differs: worker={worker_cfg.get(key)!r} vs shared={shared_cfg.get(key)!r}" def test_nested_vault_config_respected(self, tmp_vault: Path) -> None: """Nested vault_config block takes precedence over top-level keys.""" @@ -126,9 +127,7 @@ class TestWorkerPipelinePaths: for key in expected_keys: assert key in paths, f"Missing key: {key}" - def test_pipeline_paths_values_from_shared_resolver( - self, tmp_vault: Path - ) -> None: + def test_pipeline_paths_values_from_shared_resolver(self, tmp_vault: Path) -> None: """Verify shared resolver keys have correct values.""" from paperforge.config import paperforge_paths as shared_paths from paperforge.worker.sync import pipeline_paths @@ -137,9 +136,7 @@ class TestWorkerPipelinePaths: worker = pipeline_paths(tmp_vault) for key in ["exports", "ocr", "library_records", "literature", "control", "bases"]: - assert worker[key] == shared[key], ( - f"Key '{key}' differs: worker={worker[key]} vs shared={shared[key]}" - ) + assert worker[key] == shared[key], f"Key '{key}' differs: worker={worker[key]} vs shared={shared[key]}" class TestLegacyStatusSubprocess: @@ -159,10 +156,7 @@ class TestLegacyStatusSubprocess: env=env, ) # Should exit 0, no import errors - assert result.returncode == 0, ( - f"status command failed with code {result.returncode}\n" - f"stdout: {result.stdout}\nstderr: {result.stderr}" - ) - assert "ImportError" not in result.stderr, ( - f"Import error in stderr: {result.stderr}" - ) + assert ( + result.returncode == 0 + ), f"status command failed with code {result.returncode}\nstdout: {result.stdout}\nstderr: {result.stderr}" + assert "ImportError" not in result.stderr, f"Import error in stderr: {result.stderr}"