feat(phase-12): extract worker modules from literature_pipeline.py

- Create paperforge/worker/ package with sync.py, ocr.py, repair.py,
  status.py, deep_reading.py, update.py, base_views.py
- Update paperforge/commands/*.py to import from new worker modules
- Update paperforge/cli.py _import_worker_functions for new paths
- Update paperforge/config.py ld_deep_script resolution with fallback
- Update setup_wizard.py for new worker/skills paths
- Add function-level imports to break circular dependencies
This commit is contained in:
Research Assistant 2026-04-24 22:32:25 +08:00
parent dca498e85b
commit 498a9edfe5
16 changed files with 5329 additions and 30 deletions

View file

@ -86,15 +86,12 @@ def _import_worker_functions() -> None:
global run_status, run_selection_sync, run_index_refresh
global run_deep_reading, run_repair, run_ocr, ensure_base_views
from pipeline.worker.scripts.literature_pipeline import (
run_status as _rs,
run_selection_sync as _rss,
run_index_refresh as _rir,
run_deep_reading as _rdr,
run_repair as _rr,
run_ocr as _ro,
ensure_base_views as _ebu,
)
from paperforge.worker.status import run_status as _rs
from paperforge.worker.sync import run_selection_sync as _rss, run_index_refresh as _rir
from paperforge.worker.deep_reading import run_deep_reading as _rdr
from paperforge.worker.repair import run_repair as _rr
from paperforge.worker.ocr import run_ocr as _ro
from paperforge.worker.base_views import ensure_base_views as _ebu
if run_status is None:
run_status = _rs
@ -337,7 +334,7 @@ def main(argv: list[str] | None = None) -> int:
return 0
if args.command == "doctor":
from pipeline.worker.scripts.literature_pipeline import run_doctor
from paperforge.worker.status import run_doctor
return run_doctor(vault)
print(f"Error: unknown command {args.command}", file=sys.stderr)

View file

@ -19,7 +19,7 @@ def _get_run_deep_reading():
repo_root = Path(__file__).resolve().parent.parent.parent
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from pipeline.worker.scripts.literature_pipeline import run_deep_reading
from paperforge.worker.deep_reading import run_deep_reading
return run_deep_reading

View file

@ -40,7 +40,7 @@ def _get_run_ocr():
repo_root = Path(__file__).resolve().parent.parent.parent
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from pipeline.worker.scripts.literature_pipeline import run_ocr
from paperforge.worker.ocr import run_ocr
return run_ocr

View file

@ -19,7 +19,7 @@ def _get_run_repair():
repo_root = Path(__file__).resolve().parent.parent.parent
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from pipeline.worker.scripts.literature_pipeline import run_repair
from paperforge.worker.repair import run_repair
return run_repair

View file

@ -23,7 +23,7 @@ def _get_run_status():
repo_root = Path(__file__).resolve().parent.parent.parent
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from pipeline.worker.scripts.literature_pipeline import run_status
from paperforge.worker.status import run_status
return run_status

View file

@ -19,7 +19,7 @@ def _get_run_selection_sync():
repo_root = Path(__file__).resolve().parent.parent.parent
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from pipeline.worker.scripts.literature_pipeline import run_selection_sync
from paperforge.worker.sync import run_selection_sync
return run_selection_sync
@ -39,7 +39,7 @@ def _get_run_index_refresh():
repo_root = Path(__file__).resolve().parent.parent.parent
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from pipeline.worker.scripts.literature_pipeline import run_index_refresh
from paperforge.worker.sync import run_index_refresh
return run_index_refresh

View file

@ -256,8 +256,17 @@ def paperforge_paths(
# Resolve script paths relative to vault (for portability in copied installs)
# worker_script: look relative to vault root
worker_script = vault / "pipeline" / "worker" / "scripts" / "literature_pipeline.py"
# ld_deep_script: look relative to skill_dir
# ld_deep_script: look relative to skill_dir first, then repo paperforge/skills for dev
ld_deep_script = skill_path / "literature-qa" / "scripts" / "ld_deep.py"
if not ld_deep_script.exists():
repo_skill = Path(__file__).parent / "skills" / "literature-qa" / "scripts" / "ld_deep.py"
if repo_skill.exists():
ld_deep_script = repo_skill
else:
# Backward compat: old skills/ location during transition
old_repo_skill = Path(__file__).parent.parent / "skills" / "literature-qa" / "scripts" / "ld_deep.py"
if old_repo_skill.exists():
ld_deep_script = old_repo_skill
return {
"vault": vault,

View file

@ -0,0 +1,34 @@
"""PaperForge worker modules — migrated from pipeline/worker/scripts."""
from paperforge.worker.sync import (
load_export_rows,
run_selection_sync,
run_index_refresh,
obsidian_wikilink_for_pdf,
absolutize_vault_path,
_normalize_attachment_path,
_identify_main_pdf,
)
from paperforge.worker.ocr import run_ocr
from paperforge.worker.repair import run_repair
from paperforge.worker.status import run_doctor, run_status
from paperforge.worker.deep_reading import run_deep_reading
from paperforge.worker.base_views import build_base_views, merge_base_views, ensure_base_views
__all__ = [
"load_export_rows",
"run_selection_sync",
"run_index_refresh",
"run_ocr",
"run_repair",
"run_doctor",
"run_status",
"run_deep_reading",
"build_base_views",
"merge_base_views",
"ensure_base_views",
"obsidian_wikilink_for_pdf",
"absolutize_vault_path",
"_normalize_attachment_path",
"_identify_main_pdf",
]

View file

@ -0,0 +1,516 @@
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
root = shared["paperforge"]
control_root = shared["control"]
return {
**shared,
# Worker-only keys (added on top of shared resolver output)
"pipeline": root,
"candidates": root / "candidates" / "candidates.json",
"candidate_inbox": root / "candidates" / "inbox",
"candidate_archive": root / "candidates" / "archive",
"search_tasks": root / "search" / "tasks",
"search_archive": root / "search" / "archive",
"search_results": root / "search" / "results",
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
"records": control_root / "candidate-records",
"review": root / "candidates" / "review-latest.md",
"config": root / "config" / "domain-collections.json",
"queue": root / "writeback" / "writeback-queue.jsonl",
"log": root / "writeback" / "writeback-log.jsonl",
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
"index": root / "indexes" / "formal-library.json",
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
known_exports.add(export_path.name)
changed = True
if changed:
config_path.parent.mkdir(parents=True, exist_ok=True)
write_json(config_path, config)
return config
def base_markdown_filter(path: Path, vault: Path) -> str:
try:
return str(path.relative_to(vault)).replace('\\', '/')
except ValueError:
return str(path).replace('\\', '/')
PAPERFORGE_VIEW_PREFIX = "# PAPERFORGE_VIEW: "
def build_base_views(domain: str) -> list[dict]:
"""Build the 8-view list for a domain Base file.
Args:
domain: The domain name (e.g., "骨科") passed for compatibility but each view has fixed name/filter.
Returns:
List of 8 view dicts, each with keys: name (str), order (list), filter (str|None).
"""
return [
{
"name": "控制面板",
"order": ["file.name", "title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
"filter": None,
},
{
"name": "推荐分析",
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
"filter": "analyze = true AND recommend_analyze = true",
},
{
"name": "待 OCR",
"order": ["year", "title", "has_pdf", "do_ocr", "ocr_status", "pdf_path"],
"filter": 'do_ocr = true AND ocr_status = "pending"',
},
{
"name": "OCR 完成",
"order": ["year", "title", "has_pdf", "do_ocr", "ocr_status", "pdf_path"],
"filter": 'ocr_status = "done"',
},
{
"name": "待深度阅读",
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"],
"filter": 'analyze = true AND ocr_status = "done" AND deep_reading_status = "pending"',
},
{
"name": "深度阅读完成",
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"],
"filter": 'deep_reading_status = "done"',
},
{
"name": "正式卡片",
"order": ["title", "year", "has_pdf", "deep_reading_status", "pdf_path"],
"filter": 'deep_reading_status = "done"',
},
{
"name": "全记录",
"order": ["title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
"filter": None,
},
]
def substitute_config_placeholders(content: str, paths: dict[str, Path]) -> str:
"""Replace ${SCREAMING_SNAKE_CASE} path placeholders with vault-relative paths.
Args:
content: The Base file content string with ${PLACEHOLDER} tokens.
paths: dict of path key -> Path objects (from paperforge_paths()).
Returns:
Content with placeholders replaced by vault-relative paths.
Unrecognized placeholders are left unchanged.
"""
substitutions = {
"LIBRARY_RECORDS": paths.get("library_records"),
"LITERATURE": paths.get("literature"),
"CONTROL_DIR": paths.get("control"),
}
result = content
for key, path in substitutions.items():
if path is not None:
vault = paths.get("vault")
if vault is not None:
try:
rel = path.relative_to(vault)
result = result.replace(f"${{{key}}}", str(rel).replace(chr(92), "/"))
except ValueError:
result = result.replace(f"${{{key}}}", str(path).replace(chr(92), "/"))
return result
def _render_views_section(views: list[dict]) -> str:
"""Render a list of view dicts to YAML views: section."""
lines = []
for v in views:
lines.append(f" - type: table")
lines.append(f' name: "{v["name"]}"')
lines.append(f" order:")
for col in v["order"]:
lines.append(f" - {col}")
if v["filter"]:
lines.append(f' filter: \'{v["filter"]}\'')
return "\n".join(lines)
def merge_base_views(existing_content: str | None, new_views: list[dict]) -> str:
"""Incrementally merge standard PaperForge views into an existing .base file.
Strategy:
- PaperForge generates exactly 8 views with known names (STANDARD_VIEW_NAMES).
- Any OTHER views in the existing file are user-defined and MUST be preserved.
- Each PaperForge view is preceded by a PAPERFORGE_VIEW_PREFIX comment marker.
- On refresh: replace ALL PaperForge views (identified by prefix) with fresh ones.
- User views (no prefix) are left completely untouched.
Args:
existing_content: Raw text of existing .base file (or None/empty for fresh generation).
new_views: List of 8 view dicts from build_base_views().
Returns:
Merged .base file content with PaperForge views updated, user views preserved.
"""
PROPERTIES_YAML = """properties:
zotero_key:
displayName: "Zotero Key"
title:
displayName: "Title"
year:
displayName: "Year"
has_pdf:
displayName: "PDF"
do_ocr:
displayName: "OCR"
analyze:
displayName: "Analyze"
ocr_status:
displayName: "OCR Status"
deep_reading_status:
displayName: "Deep Reading"
pdf_path:
displayName: "PDF Path"
fulltext_md_path:
displayName: "Fulltext"
"""
if not existing_content or not existing_content.strip():
fresh_views_yaml = _render_views_section(new_views)
return f"""filters:
and:
- file.inFolder("{new_views[0]['name']}")
{PROPERTIES_YAML}
views:
{fresh_views_yaml}"""
lines = existing_content.split("\n")
views_start_idx = None
for i, line in enumerate(lines):
if line.strip().startswith("views:"):
views_start_idx = i
break
if views_start_idx is None:
fresh_views_yaml = _render_views_section(new_views)
return f"""{existing_content}
# --- PaperForge views regenerated (views: section was missing) ---
{PROPERTIES_YAML}
views:
{fresh_views_yaml}"""
header_lines = lines[:views_start_idx + 1]
new_pf_blocks = []
for v in new_views:
rendered = f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n"
rendered += f" - type: table\n"
rendered += f' name: "{v["name"]}"\n'
rendered += f" order:\n"
for col in v["order"]:
rendered += f" - {col}\n"
if v["filter"]:
rendered += f' filter: \'{v["filter"]}\'\n'
else:
rendered += '\n'
new_pf_blocks.append((v["name"], rendered))
rebuilt_views_lines = []
pf_names_seen = set()
i = views_start_idx + 1
pending_pf_view_name = None
while i < len(lines):
line = lines[i]
if line.startswith(PAPERFORGE_VIEW_PREFIX):
pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX):].strip()
pf_names_seen.add(pending_pf_view_name)
i += 1
continue
elif pending_pf_view_name is not None and line.strip().startswith("- type: table"):
pf_block = next((b for n, b in new_pf_blocks if n == pending_pf_view_name), None)
if pf_block:
rebuilt_views_lines.append(pf_block)
pending_pf_view_name = None
i += 1
continue
elif line.strip().startswith("- type: table"):
pending_pf_view_name = None
view_block_lines = [line]
i += 1
while i < len(lines):
next_line = lines[i]
if next_line.strip().startswith(PAPERFORGE_VIEW_PREFIX):
break
if next_line.strip().startswith("- type: table"):
break
if next_line.strip() and not next_line.startswith(" ") and not next_line.startswith("\t"):
break
view_block_lines.append(next_line)
i += 1
rebuilt_views_lines.append("\n".join(view_block_lines))
continue
else:
pending_pf_view_name = None
i += 1
for view_name, pf_block in new_pf_blocks:
if view_name not in pf_names_seen:
rebuilt_views_lines.append(pf_block)
result_lines = header_lines + rebuilt_views_lines
return "\n".join(result_lines)
def _build_base_yaml(folder_filter: str, views: list[dict]) -> str:
"""Build complete .base YAML with PAPERFORGE_VIEW_PREFIX markers on each view."""
views_yaml = ""
for v in views:
views_yaml += f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n"
views_yaml += f" - type: table\n"
views_yaml += f' name: "{v["name"]}"\n'
views_yaml += f" order:\n"
for col in v["order"]:
views_yaml += f" - {col}\n"
if v["filter"]:
views_yaml += f' filter: \'{v["filter"]}\'\n'
else:
views_yaml += '\n'
views_yaml = views_yaml.rstrip('\n')
return f"""filters:
and:
- file.inFolder("{folder_filter}")
properties:
zotero_key:
displayName: "Zotero Key"
title:
displayName: "Title"
year:
displayName: "Year"
has_pdf:
displayName: "PDF"
do_ocr:
displayName: "OCR"
analyze:
displayName: "Analyze"
ocr_status:
displayName: "OCR Status"
deep_reading_status:
displayName: "Deep Reading"
pdf_path:
displayName: "PDF Path"
fulltext_md_path:
displayName: "Fulltext"
views:
{views_yaml}"""
def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: bool = False) -> None:
"""Generate/refresh Domain Base files with incremental merge (preserves user views).
Each PaperForge standard view is marked with PAPERFORGE_VIEW_PREFIX. On refresh,
only PaperForge views are replaced; user-defined views are preserved.
"""
paths["bases"].mkdir(parents=True, exist_ok=True)
def refresh_base(base_path: Path, folder_filter: str, views: list[dict]) -> None:
"""Refresh a single .base file: merge PaperForge views, preserve user views."""
if base_path.exists() and not force:
existing = base_path.read_text(encoding="utf-8")
merged = merge_base_views(existing, views)
else:
merged = _build_base_yaml(folder_filter, views)
merged = substitute_config_placeholders(merged, paths)
base_path.write_text(merged, encoding="utf-8")
seen_domains = set()
for entry in config.get("domains", []):
domain = str(entry.get("domain", "") or "").strip()
if not domain or domain in seen_domains:
continue
seen_domains.add(domain)
domain_views = build_base_views(domain)
folder_filter = f"${{LIBRARY_RECORDS}}/{domain}"
base_path = paths["bases"] / f"{slugify_filename(domain)}.base"
refresh_base(base_path, folder_filter, domain_views)
hub_views = build_base_views("Literature Hub")
hub_path = paths["bases"] / "Literature Hub.base"
refresh_base(hub_path, "${LIBRARY_RECORDS}", hub_views)
all_views = build_base_views("All Records")
pf_base = paths["bases"] / "PaperForge.base"
refresh_base(pf_base, "${LIBRARY_RECORDS}", all_views)

View file

@ -0,0 +1,324 @@
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.worker.sync import has_deep_reading_content, load_export_rows
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.ocr import validate_ocr_meta
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
root = shared["paperforge"]
control_root = shared["control"]
return {
**shared,
# Worker-only keys (added on top of shared resolver output)
"pipeline": root,
"candidates": root / "candidates" / "candidates.json",
"candidate_inbox": root / "candidates" / "inbox",
"candidate_archive": root / "candidates" / "archive",
"search_tasks": root / "search" / "tasks",
"search_archive": root / "search" / "archive",
"search_results": root / "search" / "results",
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
"records": control_root / "candidate-records",
"review": root / "candidates" / "review-latest.md",
"config": root / "config" / "domain-collections.json",
"queue": root / "writeback" / "writeback-queue.jsonl",
"log": root / "writeback" / "writeback-log.jsonl",
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
"index": root / "indexes" / "formal-library.json",
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
known_exports.add(export_path.name)
changed = True
if changed:
config_path.parent.mkdir(parents=True, exist_ok=True)
write_json(config_path, config)
return config
def _resolve_formal_note_path(vault: Path, zotero_key: str, domain: str) -> Path | None:
"""Resolve formal literature note by zotero_key."""
lit_root = pipeline_paths(vault)['literature']
domain_dir = lit_root / domain
if not domain_dir.exists():
return None
frontmatter_pattern = re.compile(f'^\\s*zotero_key:\\s*"?{re.escape(zotero_key)}"?\\s*$', re.MULTILINE)
for note_path in domain_dir.rglob('*.md'):
try:
text = note_path.read_text(encoding='utf-8')
except UnicodeDecodeError:
text = note_path.read_text(encoding='utf-8', errors='ignore')
if frontmatter_pattern.search(text):
return note_path
return None
def run_deep_reading(vault: Path, verbose: bool = False) -> int:
"""Sync deep-reading status between formal notes and library records.
This worker does NOT generate content. It only:
1. Scans formal literature notes for `## 🔍 精读` content
2. Updates library-records/*.md frontmatter to match actual state
3. Reports the queue of papers awaiting deep reading
Actual content filling is done via /pf-deep (agent-driven).
"""
paths = pipeline_paths(vault)
config = load_domain_config(paths)
ensure_base_views(vault, paths, config)
domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']}
synced = 0
pending_queue: list[dict] = []
for export_path in sorted(paths['exports'].glob('*.json')):
domain = domain_lookup.get(export_path.name, export_path.stem)
for item in load_export_rows(export_path):
key = item['key']
record_dir = paths['library_records'] / domain
record_path = record_dir / f'{key}.md'
if not record_path.exists():
continue
record_text = record_path.read_text(encoding='utf-8')
analyze_match = re.search('^analyze:\\s*(true|false)$', record_text, re.MULTILINE)
is_analyze = analyze_match and analyze_match.group(1) == 'true'
do_ocr_match = re.search('^do_ocr:\\s*(true|false)$', record_text, re.MULTILINE)
is_do_ocr = do_ocr_match and do_ocr_match.group(1) == 'true'
note_path = _resolve_formal_note_path(vault, key, domain)
has_content = False
if note_path and note_path.exists():
note_text = note_path.read_text(encoding='utf-8')
has_content = has_deep_reading_content(note_text)
correct_status = 'done' if has_content else 'pending'
status_match = re.search('^deep_reading_status:\\s*"??"?$', record_text, re.MULTILINE)
current_status = status_match.group(1) if status_match else 'pending'
if current_status != correct_status:
new_text = re.sub('^deep_reading_status:\\s*"?.*?"?$', f'deep_reading_status: {yaml_quote(correct_status)}', record_text, flags=re.MULTILINE, count=1)
record_path.write_text(new_text, encoding='utf-8')
synced += 1
if is_analyze and correct_status == 'pending':
meta_path = paths['ocr'] / key / 'meta.json'
ocr_status = 'pending'
if meta_path.exists():
try:
meta = read_json(meta_path)
validated_status, error_msg = validate_ocr_meta(paths, meta)
ocr_status = validated_status
except Exception:
pass
pending_queue.append({
'zotero_key': key,
'domain': domain,
'title': item.get('title', ''),
'ocr_status': ocr_status,
'is_analyze': is_analyze,
'is_do_ocr': is_do_ocr,
})
if pending_queue:
ready = [q for q in pending_queue if q['ocr_status'] == 'done']
waiting = [q for q in pending_queue if q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing')]
blocked = [q for q in pending_queue if q['is_analyze'] and q['ocr_status'] not in ('done', '') and not (q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing'))]
report_lines = ['# 待精读队列', '']
if ready:
report_lines.extend([f'## 就绪 ({len(ready)} 篇) — OCR 已完成,可直接 /pf-deep', ''])
for q in ready:
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']}")
report_lines.append('')
if waiting:
report_lines.extend([f'## 等待 OCR ({len(waiting)} 篇)', ''])
for q in waiting:
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status']}")
report_lines.append('')
if blocked:
report_lines.extend([f'## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR', ''])
for q in blocked:
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}")
report_lines.append('')
if verbose:
report_lines.append('### 修复步骤\n')
for q in blocked:
ocr_s = q['ocr_status'] or ''
if not ocr_s or ocr_s == 'pending':
fix = f"paperforge ocr"
report_lines.append(f"- `{q['zotero_key']}`: 运行 `{fix}` 启动 OCR")
elif ocr_s == 'processing':
report_lines.append(f"- `{q['zotero_key']}`: OCR 进行中,请等待完成")
elif ocr_s == 'failed':
report_lines.append(f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息,然后重新运行 `paperforge ocr`")
else:
report_lines.append(f"- `{q['zotero_key']}`: 运行 `paperforge ocr` 重试")
report_lines.append('')
report_lines.extend(['## 操作', '', '- 对就绪论文,使用 `/pf-deep <zotero_key>` 触发精读', '- 批量触发:提供多个 key用 subagent 并行处理', ''])
else:
report_lines = ['# 待精读队列', '', '所有 analyze=true 的论文已完成精读。', '']
report_path = paths['pipeline'] / 'deep-reading-queue.md'
report_path.write_text('\n'.join(report_lines), encoding='utf-8')
print(f'deep-reading: synced {synced} records, {len(pending_queue)} pending')
return 0

1376
paperforge/worker/ocr.py Normal file

File diff suppressed because it is too large Load diff

548
paperforge/worker/repair.py Normal file
View file

@ -0,0 +1,548 @@
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.worker.sync import (
load_export_rows,
obsidian_wikilink_for_pdf,
update_frontmatter_field,
)
from paperforge.worker.deep_reading import _resolve_formal_note_path
from paperforge.worker.ocr import validate_ocr_meta
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
root = shared["paperforge"]
control_root = shared["control"]
return {
**shared,
# Worker-only keys (added on top of shared resolver output)
"pipeline": root,
"candidates": root / "candidates" / "candidates.json",
"candidate_inbox": root / "candidates" / "inbox",
"candidate_archive": root / "candidates" / "archive",
"search_tasks": root / "search" / "tasks",
"search_archive": root / "search" / "archive",
"search_results": root / "search" / "results",
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
"records": control_root / "candidate-records",
"review": root / "candidates" / "review-latest.md",
"config": root / "config" / "domain-collections.json",
"queue": root / "writeback" / "writeback-queue.jsonl",
"log": root / "writeback" / "writeback-log.jsonl",
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
"index": root / "indexes" / "formal-library.json",
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
known_exports.add(export_path.name)
changed = True
if changed:
config_path.parent.mkdir(parents=True, exist_ok=True)
write_json(config_path, config)
return config
def _find_export_for_domain(paths: dict[str, Path], domain: str) -> Path | None:
"""Find the BBT export JSON file for a given domain."""
for export_path in sorted(paths["exports"].glob("*.json")):
if export_path.stem == domain:
return export_path
return None
def _detect_path_errors(paths: dict[str, Path], verbose: bool = False) -> dict:
"""Scan library-records for path_error fields.
Returns dict with:
- total: total count of records with path_error
- by_type: dict mapping error type -> count
- records: list of record dicts with keys: path, zotero_key, domain, path_error, bbt_path_raw
"""
result: dict = {"total": 0, "by_type": {}, "records": []}
if not paths["library_records"].exists():
return result
for record_path in paths["library_records"].rglob("*.md"):
try:
text = record_path.read_text(encoding="utf-8")
except Exception as e:
if verbose:
print(f"[repair] error reading {record_path}: {e}")
continue
err_match = re.search(r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE)
if not err_match:
continue
path_error = err_match.group(1).strip()
if not path_error:
continue
key_match = re.search(r'^zotero_key:\s*"?(.+?)"?\s*$', text, re.MULTILINE)
if not key_match:
continue
zotero_key = key_match.group(1).strip()
domain = record_path.parent.name
bbt_match = re.search(r'^bbt_path_raw:\s*"(.*?)"\s*$', text, re.MULTILINE)
bbt_path_raw = bbt_match.group(1) if bbt_match else ""
result["total"] += 1
result["by_type"][path_error] = result["by_type"].get(path_error, 0) + 1
result["records"].append(
{
"path": record_path,
"zotero_key": zotero_key,
"domain": domain,
"path_error": path_error,
"bbt_path_raw": bbt_path_raw,
"text": text,
}
)
return result
def repair_pdf_paths(
vault: Path,
paths: dict[str, Path],
error_records: list[dict],
verbose: bool = False,
) -> int:
"""Re-resolve PDF paths for items with path_error.
Returns number of paths successfully fixed.
"""
fixed = 0
from paperforge.pdf_resolver import resolve_pdf_path
cfg = load_vault_config(vault)
zotero_dir = vault / cfg.get("system_dir", "99_System") / "Zotero"
# Cache export rows by domain to avoid reloading
domain_exports: dict[str, list[dict]] = {}
for record in error_records:
record_path = record["path"]
zotero_key = record["zotero_key"]
domain = record["domain"]
path_error = record["path_error"]
text = record["text"]
# For not_found errors, try to find the item in BBT export and re-process
if path_error == "not_found":
export_rows = domain_exports.get(domain)
if export_rows is None:
export_path = _find_export_for_domain(paths, domain)
if export_path and export_path.exists():
try:
export_rows = load_export_rows(export_path)
domain_exports[domain] = export_rows
except Exception as e:
if verbose:
print(
f"[repair] error loading export for {domain}: {e}"
)
export_rows = []
else:
export_rows = []
domain_exports[domain] = export_rows
item = next((r for r in export_rows if r["key"] == zotero_key), None)
if item:
pdf_path = item.get("pdf_path", "")
if pdf_path:
new_wikilink = obsidian_wikilink_for_pdf(
pdf_path, vault, zotero_dir
)
if new_wikilink:
new_text = update_frontmatter_field(
text, "pdf_path", new_wikilink
)
new_text = update_frontmatter_field(
new_text, "path_error", ""
)
new_text = update_frontmatter_field(
new_text,
"bbt_path_raw",
item.get("bbt_path_raw", ""),
)
new_text = update_frontmatter_field(
new_text,
"zotero_storage_key",
item.get("zotero_storage_key", ""),
)
if new_text != text:
record_path.write_text(new_text, encoding="utf-8")
fixed += 1
if verbose:
print(
f"[repair] fixed path for {zotero_key}: {new_wikilink}"
)
continue
# For all errors, try resolving the current pdf_path
pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE)
if pdf_match:
current_pdf = pdf_match.group(1).strip()
if current_pdf:
raw_path = current_pdf.strip("[]")
resolved = resolve_pdf_path(raw_path, True, vault, zotero_dir)
if resolved:
new_text = update_frontmatter_field(text, "path_error", "")
if new_text != text:
record_path.write_text(new_text, encoding="utf-8")
fixed += 1
if verbose:
print(f"[repair] cleared path_error for {zotero_key}")
else:
if verbose:
print(f"[repair] {zotero_key} path still unresolved")
else:
if verbose:
print(
f"[repair] {zotero_key} has empty pdf_path (not_found)"
)
else:
if verbose:
print(f"[repair] {zotero_key} has no pdf_path field")
return fixed
def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = False, fix_paths: bool = False) -> dict:
"""Scan all domains for three-way state divergence and optionally repair.
Compares three sources of ocr_status:
1. library_record.md frontmatter ocr_status
2. formal_note.md frontmatter ocr_status
3. meta.json ocr_status (post-validate_ocr_meta())
Returns:
dict with scanned, divergent, fixed, errors counts
"""
result = {"scanned": 0, "divergent": [], "fixed": 0, "errors": []}
config = load_domain_config(paths)
domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']}
record_paths = list(paths['library_records'].rglob('*.md'))
for record_path in record_paths:
try:
record_text = record_path.read_text(encoding='utf-8')
except Exception as e:
result['errors'].append({"file": str(record_path), "error": str(e)})
continue
key_match = re.search('^zotero_key:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE)
if not key_match:
continue
zotero_key = key_match.group(1).strip()
domain = record_path.parent.name
record_dir = record_path.parent
result['scanned'] += 1
lib_ocr_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE)
lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else 'pending'
note_path = _resolve_formal_note_path(vault, zotero_key, domain)
note_ocr_status = None
if note_path and note_path.exists():
try:
note_text = note_path.read_text(encoding='utf-8')
note_status_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', note_text, re.MULTILINE)
note_ocr_status = note_status_match.group(1).strip() if note_status_match else None
except Exception:
pass
meta_path = paths['ocr'] / zotero_key / 'meta.json'
meta_ocr_status = None
meta_validated_status = None
validated_status = None
validated_error = ""
if meta_path.exists():
try:
meta = read_json(meta_path)
validated_status, validated_error = validate_ocr_meta(paths, meta)
meta_validated_status = validated_status
if validated_error and verbose:
print(f"[repair] {zotero_key} meta validation error: {validated_error}")
raw_status = str(meta.get('ocr_status', '') or '').strip().lower()
meta_ocr_status = raw_status if raw_status else None
if meta_validated_status == 'done_incomplete':
meta_ocr_status = 'done_incomplete'
except Exception as e:
result['errors'].append({"file": str(meta_path), "error": str(e)})
meta_ocr_status = None
is_divergent = False
div_reason = ""
if meta_validated_status == 'done_incomplete':
is_divergent = True
div_reason = f"meta validation: done_incomplete ({validated_error})"
elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing', None):
is_divergent = True
div_reason = f"library_record done but meta {meta_ocr_status or 'missing'}"
elif note_ocr_status == 'done' and (meta_ocr_status is None or meta_validated_status == 'done_incomplete'):
is_divergent = True
div_reason = "formal_note done but meta.json missing/invalid"
elif lib_ocr_status != 'pending' and meta_ocr_status is not None and meta_validated_status is not None and lib_ocr_status != meta_validated_status:
is_divergent = True
div_reason = f"library_record={lib_ocr_status} vs meta post-validation={meta_validated_status}"
if is_divergent:
item = {
"zotero_key": zotero_key,
"domain": domain,
"library_record_ocr_status": lib_ocr_status,
"formal_note_ocr_status": note_ocr_status,
"meta_ocr_status": meta_validated_status or meta_ocr_status,
"reason": div_reason,
}
result['divergent'].append(item)
if verbose:
print(f"[repair] divergent: {zotero_key} | {div_reason}")
if fix:
fixed_library_record = False
fixed_formal_note = False
fixed_meta = False
new_status = 'pending'
if meta_ocr_status is None or meta_validated_status == 'done_incomplete':
new_status = 'pending'
new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status)
if new_record_text != record_text:
record_path.write_text(new_record_text, encoding='utf-8')
fixed_library_record = True
if note_path and note_path.exists():
try:
note_text = note_path.read_text(encoding='utf-8')
new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status)
if new_note_text != note_text:
note_path.write_text(new_note_text, encoding='utf-8')
fixed_formal_note = True
except Exception:
pass
if meta_validated_status is not None and meta_validated_status != 'done':
if meta_path.exists():
try:
meta = read_json(meta_path)
meta['ocr_status'] = 'pending'
write_json(meta_path, meta)
fixed_meta = True
except Exception:
pass
record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE)
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true'
if not is_do_ocr:
final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true')
if final_record_text != new_record_text:
record_path.write_text(final_record_text, encoding='utf-8')
fixed_library_record = True
elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing'):
new_status = 'pending'
new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status)
if new_record_text != record_text:
record_path.write_text(new_record_text, encoding='utf-8')
fixed_library_record = True
if note_path and note_path.exists():
try:
note_text = note_path.read_text(encoding='utf-8')
new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status)
if new_note_text != note_text:
note_path.write_text(new_note_text, encoding='utf-8')
fixed_formal_note = True
except Exception:
pass
if meta_path.exists():
try:
meta = read_json(meta_path)
meta['ocr_status'] = 'pending'
write_json(meta_path, meta)
fixed_meta = True
except Exception:
pass
record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE)
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true'
if not is_do_ocr:
final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true')
if final_record_text != new_record_text:
record_path.write_text(final_record_text, encoding='utf-8')
fixed_library_record = True
fixed_count = sum([fixed_library_record, fixed_formal_note, fixed_meta])
result['fixed'] += fixed_count
if verbose and fixed_count > 0:
print(f"[repair] fixed {fixed_count} files for {zotero_key}")
# Path error detection and repair
path_errors = _detect_path_errors(paths, verbose)
if path_errors["total"] > 0:
error_summary = ", ".join(
f"{count} {err}" for err, count in sorted(path_errors["by_type"].items())
)
print(
f"[repair] Found {path_errors['total']} items with path errors: {error_summary}"
)
if fix_paths:
fixed_count = repair_pdf_paths(
vault, paths, path_errors["records"], verbose
)
print(f"[repair] Fixed {fixed_count} PDF paths")
else:
print("[repair] Tip: run with --fix-paths to attempt auto-resolution")
elif verbose:
print("[repair] No path errors found")
result["path_errors"] = path_errors
return result

631
paperforge/worker/status.py Normal file
View file

@ -0,0 +1,631 @@
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.worker.base_views import ensure_base_views
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
root = shared["paperforge"]
control_root = shared["control"]
return {
**shared,
# Worker-only keys (added on top of shared resolver output)
"pipeline": root,
"candidates": root / "candidates" / "candidates.json",
"candidate_inbox": root / "candidates" / "inbox",
"candidate_archive": root / "candidates" / "archive",
"search_tasks": root / "search" / "tasks",
"search_archive": root / "search" / "archive",
"search_results": root / "search" / "results",
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
"records": control_root / "candidate-records",
"review": root / "candidates" / "review-latest.md",
"config": root / "config" / "domain-collections.json",
"queue": root / "writeback" / "writeback-queue.jsonl",
"log": root / "writeback" / "writeback-log.jsonl",
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
"index": root / "indexes" / "formal-library.json",
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
known_exports.add(export_path.name)
changed = True
if changed:
config_path.parent.mkdir(parents=True, exist_ok=True)
write_json(config_path, config)
return config
def _detect_zotero_data_dir() -> str | None:
"""Try to detect the user's Zotero data directory."""
if os.name == "nt":
appdata = os.environ.get("APPDATA", "")
if appdata:
candidate = Path(appdata) / "Zotero"
if candidate.exists() and (candidate / "storage").exists():
return str(candidate)
home = Path.home()
candidates = []
if os.name == "nt":
candidates = [
home / "Zotero",
home / "AppData" / "Roaming" / "Zotero",
]
else:
candidates = [
home / "Zotero",
home / ".zotero" / "zotero",
home / "Library" / "Application Support" / "Zotero",
]
for candidate in candidates:
if candidate.exists() and (candidate / "storage").exists():
return str(candidate)
return None
def check_zotero_location(vault: Path, cfg: dict, add_check) -> None:
"""Detect if Zotero data directory is inside vault or linked via junction."""
system_dir = vault / cfg["system_dir"]
zotero_link = system_dir / "Zotero"
if zotero_link.exists():
if zotero_link.is_symlink() or _is_junction(zotero_link):
try:
target = os.path.realpath(zotero_link)
if Path(target).exists():
add_check(
"Path Resolution",
"pass",
f"Zotero junction valid -> {target}",
)
else:
add_check(
"Path Resolution",
"warn",
f"Zotero junction target missing: {target}",
f"Recreate junction: mklink /J \"{zotero_link}\" \"<Zotero数据目录>\"",
)
except Exception:
add_check(
"Path Resolution",
"warn",
"Zotero junction exists but target could not be resolved",
)
else:
if (zotero_link / "storage").exists():
add_check(
"Path Resolution",
"pass",
"Zotero inside vault -- direct paths available",
)
else:
add_check(
"Path Resolution",
"warn",
"Zotero directory exists but missing storage/ subdirectory",
"Verify this is a valid Zotero data directory",
)
else:
zotero_data_dir = _detect_zotero_data_dir()
if zotero_data_dir:
add_check(
"Path Resolution",
"warn",
"Zotero outside vault -- junction recommended",
f'Run as Administrator: mklink /J "{zotero_link}" "{zotero_data_dir}"',
)
else:
add_check(
"Path Resolution",
"fail",
"Zotero directory not found",
f'Run as Administrator: mklink /J "{zotero_link}" "<Zotero数据目录>"',
)
def _summarize_errors(errors: list[str]) -> str:
"""Summarize error types into a human-readable string."""
counts: dict[str, int] = {}
for e in errors:
counts[e] = counts.get(e, 0) + 1
return ", ".join(f"{count} {err}" for err, count in sorted(counts.items()))
def check_pdf_paths(vault: Path, paths: dict, add_check) -> None:
"""Sample up to 5 library records and validate their pdf_path wikilinks."""
record_paths = list(paths["library_records"].rglob("*.md"))
if not record_paths:
add_check("Path Resolution", "pass", "No library records to check")
return
import random
sample = (
record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5)
)
valid = 0
errors: list[str] = []
for record_path in sample:
try:
text = record_path.read_text(encoding="utf-8")
except Exception:
continue
pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE)
if not pdf_match:
continue
pdf_path = pdf_match.group(1).strip()
if not pdf_path:
continue
raw_path = pdf_path.strip("[]")
candidate = vault / raw_path.replace("/", os.sep)
if candidate.exists():
valid += 1
else:
err_match = re.search(
r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE
)
error_type = err_match.group(1) if err_match else "not_found"
errors.append(error_type)
total = len(sample)
if valid == total:
add_check("Path Resolution", "pass", f"{valid}/{total} PDF paths valid")
else:
error_summary = _summarize_errors(errors) if errors else "unknown"
add_check(
"Path Resolution",
"warn",
f"{valid}/{total} PDF paths valid, {total - valid} path errors: {error_summary}",
)
def check_wikilink_format(vault: Path, paths: dict, add_check) -> None:
"""Verify all pdf_path values in library-records use [[...]] format."""
record_paths = list(paths["library_records"].rglob("*.md"))
bad_paths: list[str] = []
for record_path in record_paths:
try:
text = record_path.read_text(encoding="utf-8")
except Exception:
continue
pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE)
if not pdf_match:
continue
pdf_path = pdf_match.group(1).strip()
if not pdf_path:
continue
if not (pdf_path.startswith("[[") and pdf_path.endswith("]]")):
bad_paths.append(str(record_path.relative_to(vault)))
if bad_paths:
add_check(
"Path Resolution",
"warn",
f"{len(bad_paths)} pdf_path values not in wikilink format",
"Re-run `paperforge sync` to regenerate wikilinks",
)
else:
add_check(
"Path Resolution",
"pass",
"All pdf_path values use wikilink format",
)
def run_doctor(vault: Path) -> int:
"""Validate PaperForge Lite setup and report by category.
Returns:
0 if all checks pass, 1 otherwise.
"""
paths = pipeline_paths(vault)
cfg = load_vault_config(vault)
checks: list[tuple[str, str, str, str]] = []
def add_check(category: str, status: str, message: str, fix: str = "") -> None:
checks.append((category, status, message, fix))
sys_version = sys.version_info
if sys_version.major >= 3 and sys_version.minor >= 10:
add_check("Python 环境", "pass", f"Python {sys_version.major}.{sys_version.minor}.{sys_version.micro}")
else:
add_check("Python 环境", "fail", f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)", "升级 Python 到 3.10 或更高版本")
required_modules = ["requests", "pymupdf", "PIL", "yaml"]
missing_modules = []
for mod in required_modules:
try:
__import__(mod)
except ImportError:
missing_modules.append(mod)
if missing_modules:
add_check("Python 环境", "fail", f"缺少模块: {', '.join(missing_modules)}", f"运行: pip install {' '.join(missing_modules)}")
elif sys_version.major >= 3 and sys_version.minor >= 10:
pass
if (vault / "paperforge.json").exists():
add_check("Vault 结构", "pass", "paperforge.json 存在")
else:
add_check("Vault 结构", "fail", "paperforge.json 不存在", "在 vault 根目录创建 paperforge.json")
system_dir = vault / cfg["system_dir"]
if system_dir.exists():
add_check("Vault 结构", "pass", f"system_dir 存在: {cfg['system_dir']}")
else:
add_check("Vault 结构", "fail", f"system_dir 不存在: {cfg['system_dir']}", f"运行: mkdir {cfg['system_dir']}")
resources_dir = vault / cfg["resources_dir"]
if resources_dir.exists():
add_check("Vault 结构", "pass", f"resources_dir 存在: {cfg['resources_dir']}")
else:
add_check("Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}")
control_dir = resources_dir / cfg.get("control_dir", "LiteratureControl")
if control_dir.exists():
add_check("Vault 结构", "pass", f"control_dir 存在: {cfg.get('control_dir', 'LiteratureControl')}")
else:
add_check("Vault 结构", "fail", f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}", f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}")
zotero_link = system_dir / "Zotero"
if zotero_link.exists():
if zotero_link.is_symlink() or _is_junction(zotero_link):
add_check("Zotero 链接", "pass", "Zotero 目录是 junction/symlink")
else:
add_check("Zotero 链接", "warn", "Zotero 目录存在但不是 junction建议使用 junction")
else:
add_check("Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} <Zotero数据目录>")
exports_dir = system_dir / "PaperForge" / "exports"
if exports_dir.exists():
add_check("BBT 导出", "pass", "exports 目录存在")
else:
add_check("BBT 导出", "fail", "exports 目录不存在", "在 Better BibTeX 设置中配置导出路径")
# Check for any valid JSON export (per-domain or library.json)
json_files = sorted(exports_dir.glob("*.json")) if exports_dir.exists() else []
valid_exports = []
for jf in json_files:
try:
data = json.loads(jf.read_text(encoding="utf-8"))
if isinstance(data, list) and len(data) > 0:
has_key = any("key" in item or "citation-key" in item for item in data[:5])
if has_key:
valid_exports.append((jf.name, len(data)))
elif isinstance(data, dict) and isinstance(data.get("items"), list) and len(data["items"]) > 0:
has_key = any("key" in item or "citation-key" in item for item in data["items"][:5])
if has_key:
valid_exports.append((jf.name, len(data["items"])))
except (JSONDecodeError, Exception):
pass
if valid_exports:
for name, count in valid_exports:
add_check("BBT 导出", "pass", f"{name} 正常 ({count} 条)")
elif json_files:
add_check("BBT 导出", "warn", "导出文件存在但无有效 citation key")
else:
add_check("BBT 导出", "fail", "未找到 JSON 导出文件", "在 Zotero Better BibTeX 设置中配置导出路径")
env_api_key = os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN")
if env_api_key:
add_check("OCR 配置", "pass", "API Token 已配置")
else:
add_check("OCR 配置", "fail", "缺少 PADDLEOCR_API_TOKEN", "在 .env 文件中设置 PADDLEOCR_API_TOKEN")
if (paths["pipeline"] / "literature_pipeline.py").exists():
add_check("Worker 脚本", "pass", "literature_pipeline.py 存在")
try:
from paperforge.worker.sync import (
run_selection_sync, run_index_refresh,
)
from paperforge.worker.deep_reading import run_deep_reading
from paperforge.worker.ocr import run_ocr
from paperforge.worker.base_views import ensure_base_views
run_status # self-reference
add_check("Worker 脚本", "pass", "所有 worker 函数可导入")
except ImportError as e:
add_check("Worker 脚本", "fail", f"worker 函数导入失败: {e}", "检查 pipeline/worker/scripts/literature_pipeline.py")
else:
add_check("Worker 脚本", "fail", "literature_pipeline.py 不存在", "确认 PaperForge pipeline 已正确安装")
# Path Resolution checks
check_zotero_location(vault, cfg, add_check)
check_pdf_paths(vault, paths, add_check)
check_wikilink_format(vault, paths, add_check)
ld_deep_script = paths.get("ld_deep_script")
skill_dir = None
if ld_deep_script:
skill_dir = ld_deep_script.parent.parent
if skill_dir and skill_dir.exists():
# Try actual importability check
ld_deep_import_ok = False
import_error = ""
if ld_deep_script and ld_deep_script.exists():
try:
import importlib.util
spec = importlib.util.spec_from_file_location("ld_deep", ld_deep_script)
if spec and spec.loader:
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
ld_deep_import_ok = True
except Exception as e:
import_error = str(e)
if ld_deep_import_ok:
add_check("Agent 脚本", "pass", "paperforge and ld_deep importable")
else:
add_check("Agent 脚本", "warn", f"literature-qa skill 目录存在但 import 失败: {import_error}", "确认 agent_config_dir 配置正确并已运行 pip install -e .")
else:
add_check("Agent 脚本", "warn", "literature-qa skill 目录未找到", "确认 agent_config_dir 配置正确")
print("PaperForge Lite Doctor")
print("=" * 40)
current_category = ""
fix_map: dict[str, list[str]] = {}
for category, status, message, fix in checks:
if category != current_category:
if current_category:
print()
current_category = category
status_tag = {"pass": "[PASS]", "fail": "[FAIL]", "warn": "[WARN]"}[status]
print(f"{status_tag} {category}{message}")
if status == "fail" and fix:
fix_map.setdefault(category, [])
fix_map[category].append(fix)
if fix_map:
print("\n修复步骤:")
for cat, fixes in fix_map.items():
for f in fixes:
print(f" - {cat}: {f}")
print()
return 1
print()
return 0
def _is_junction(path: Path) -> bool:
"""Check if a path is a Windows junction point."""
try:
import ctypes
from ctypes import wintypes
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
GetFileAttributesW = ctypes.windll.kernel32.GetFileAttributesW
GetFileAttributesW.argtypes = [wintypes.LPCWSTR]
GetFileAttributesW.restype = wintypes.DWORD
attrs = GetFileAttributesW(str(path))
if attrs == INVALID_FILE_ATTRIBUTES:
return False
return bool(attrs & FILE_ATTRIBUTE_REPARSE_POINT)
except Exception:
return False
def run_status(vault: Path) -> int:
"""Print a compact Lite install/runtime status."""
paths = pipeline_paths(vault)
cfg = load_vault_config(vault)
config = load_domain_config(paths)
ensure_base_views(vault, paths, config)
export_files = sorted(paths['exports'].glob('*.json'))
record_count = sum(1 for _ in paths['library_records'].rglob('*.md')) if paths['library_records'].exists() else 0
note_count = sum(1 for _ in paths['literature'].rglob('*.md')) if paths['literature'].exists() else 0
base_count = sum(1 for _ in paths['bases'].glob('*.base')) if paths['bases'].exists() else 0
ocr_done = 0
ocr_total = 0
if paths['ocr'].exists():
for meta_path in paths['ocr'].glob('*/meta.json'):
ocr_total += 1
try:
meta = read_json(meta_path)
except Exception:
continue
if str(meta.get('ocr_status', '')).strip().lower() == 'done':
ocr_done += 1
env_paths = [vault / '.env', paths['pipeline'] / '.env']
env_found = [str(path.relative_to(vault)).replace('\\', '/') for path in env_paths if path.exists()]
# Count path errors
path_error_count = 0
if paths['library_records'].exists():
for record_path in paths['library_records'].rglob('*.md'):
try:
text = record_path.read_text(encoding='utf-8')
if re.search(r'^path_error:\s*"(.+?)"\s*$', text, re.MULTILINE):
path_error_count += 1
except Exception:
continue
print('PaperForge Lite status')
print(f"- vault: {vault}")
print(f"- system_dir: {cfg['system_dir']}")
print(f"- resources_dir: {cfg['resources_dir']}")
print(f"- literature_dir: {cfg['literature_dir']}")
print(f"- control_dir: {cfg['control_dir']}")
print(f"- exports: {len(export_files)} JSON file(s)")
print(f"- domains: {len(config.get('domains', []))}")
print(f"- library_records: {record_count}")
print(f"- formal_notes: {note_count}")
print(f"- bases: {base_count}")
print(f"- ocr: {ocr_done}/{ocr_total} done")
print(f"- path_errors: {path_error_count}")
if path_error_count > 0:
print(" Tip: Run `paperforge repair --fix-paths` to attempt resolution")
print(f"- env: {', '.join(env_found) if env_found else 'not configured'}")
return 0
# =============================================================================
# Update 功能
# =============================================================================
GITHUB_REPO = "LLLin000/PaperForge"
GITHUB_ZIP = f"https://github.com/{GITHUB_REPO}/archive/refs/heads/master.zip"
UPDATEABLE_PATHS = ["skills", "pipeline", "command", "scripts"]

1444
paperforge/worker/sync.py Normal file

File diff suppressed because it is too large Load diff

398
paperforge/worker/update.py Normal file
View file

@ -0,0 +1,398 @@
from __future__ import annotations
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
root = shared["paperforge"]
control_root = shared["control"]
return {
**shared,
# Worker-only keys (added on top of shared resolver output)
"pipeline": root,
"candidates": root / "candidates" / "candidates.json",
"candidate_inbox": root / "candidates" / "inbox",
"candidate_archive": root / "candidates" / "archive",
"search_tasks": root / "search" / "tasks",
"search_archive": root / "search" / "archive",
"search_results": root / "search" / "results",
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
"records": control_root / "candidate-records",
"review": root / "candidates" / "review-latest.md",
"config": root / "config" / "domain-collections.json",
"queue": root / "writeback" / "writeback-queue.jsonl",
"log": root / "writeback" / "writeback-log.jsonl",
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
"index": root / "indexes" / "formal-library.json",
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
known_exports.add(export_path.name)
changed = True
if changed:
config_path.parent.mkdir(parents=True, exist_ok=True)
write_json(config_path, config)
return config
def protected_paths(vault: Path) -> set[str]:
cfg = load_vault_config(vault)
pf = f"{cfg['system_dir']}/PaperForge"
return {
cfg["resources_dir"],
cfg["base_dir"],
f"{pf}/ocr",
f"{pf}/exports",
f"{pf}/indexes",
f"{pf}/candidates",
".env",
"AGENTS.md",
}
def _color(text: str, c: str = "") -> str:
colors = {"r": "\033[91m", "g": "\033[92m", "y": "\033[93m", "b": "\033[94m", "c": "\033[96m", "x": "\033[0m"}
if sys.platform == "win32" and not os.environ.get("FORCE_COLOR"):
return text
return f"{colors.get(c, '')}{text}{colors['x']}"
def _log(msg: str, c: str = "") -> None:
print(_color(msg, c))
def _remote_version() -> str | None:
try:
api = f"https://api.github.com/repos/{GITHUB_REPO}/contents/paperforge.json"
req = urllib.request.Request(api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"})
with urllib.request.urlopen(req, timeout=10) as resp:
data = json.loads(resp.read())
req2 = urllib.request.Request(data["download_url"], headers={"User-Agent": "PaperForge"})
with urllib.request.urlopen(req2, timeout=10) as resp2:
return json.loads(resp2.read()).get("version")
except Exception:
return None
def _scan_updates(vault: Path, source: Path) -> list[tuple[Path, Path, str]]:
updates = []
protected = protected_paths(vault)
for name in UPDATEABLE_PATHS:
src_dir = source / name
if not src_dir.exists():
continue
for src in src_dir.rglob("*"):
if not src.is_file():
continue
rel = src.relative_to(source)
dst = vault / rel
rel_str = str(rel).replace("\\", "/")
if any(rel_str.startswith(p) for p in protected):
continue
if dst.exists():
if hashlib.sha256(src.read_bytes()).hexdigest() != hashlib.sha256(dst.read_bytes()).hexdigest():
updates.append((src, dst, "UPDATE"))
else:
updates.append((src, dst, "NEW"))
return updates
def _do_backup(vault: Path, updates: list) -> Path | None:
backup_dir = vault / f".backup_{datetime.now():%Y%m%d_%H%M%S}"
backup_dir.mkdir(exist_ok=True)
count = 0
for src, dst, action in updates:
if action == "UPDATE" and dst.exists():
bp = backup_dir / dst.relative_to(vault)
bp.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(dst, bp)
count += 1
if count:
_log(f"[INFO] 已备份 {count} 个文件到 {backup_dir.name}", "c")
return backup_dir if count else None
def _apply_updates(vault: Path, updates: list) -> bool:
try:
for src, dst, action in updates:
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
return True
except Exception as e:
_log(f"[ERR] 更新失败: {e}", "r")
return False
def _rollback(vault: Path, backup_dir: Path) -> None:
_log("[INFO] 正在回滚...", "b")
for bp in backup_dir.rglob("*"):
if bp.is_file():
orig = vault / bp.relative_to(backup_dir)
orig.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(bp, orig)
_log("[OK] 回滚完成", "g")
def update_via_git(vault: Path) -> bool:
if not (vault / ".git").is_dir():
_log("[ERR] 不是 git 仓库", "r")
return False
r = subprocess.run(["git", "status", "--short"], cwd=vault, capture_output=True, text=True, encoding="utf-8")
if r.stdout.strip():
_log("[WARN] 有未提交的更改,请先提交或储藏", "y")
return False
_log("[INFO] 执行 git pull...", "b")
r = subprocess.run(["git", "pull", "origin", "master"], cwd=vault, capture_output=True, text=True, encoding="utf-8")
if r.returncode != 0:
_log(f"[ERR] git pull 失败: {r.stderr}", "r")
return False
_log("[OK] git pull 成功", "g")
if r.stdout.strip():
print(r.stdout)
return True
def update_via_zip(vault: Path) -> bool:
_log("[INFO] 下载更新包...", "b")
tmp = Path(tempfile.mkdtemp(prefix="pf_update_"))
zip_path = tmp / "update.zip"
try:
req = urllib.request.Request(GITHUB_ZIP, headers={"User-Agent": "PaperForge"})
with urllib.request.urlopen(req, timeout=60) as resp:
zip_path.write_bytes(resp.read())
with zipfile.ZipFile(zip_path, "r") as zf:
zf.extractall(tmp / "extracted")
dirs = [d for d in (tmp / "extracted").iterdir() if d.is_dir()]
source = dirs[0] if dirs else None
if not source:
_log("[ERR] 解压失败", "r")
return False
updates = _scan_updates(vault, source)
if not updates:
_log("[OK] 所有文件已是最新", "g")
return True
_log(f"\n[INFO] 发现 {len(updates)} 个文件需要更新:", "b")
for src, dst, action in updates:
_log(f" [{action}] {dst.relative_to(vault)}", "g" if action == "NEW" else "y")
backup = _do_backup(vault, updates)
if _apply_updates(vault, updates):
_log(f"\n[OK] 更新完成!共 {len(updates)} 个文件", "g")
return True
if backup:
_rollback(vault, backup)
return False
except Exception as e:
_log(f"[ERR] 下载失败: {e}", "r")
return False
finally:
shutil.rmtree(tmp, ignore_errors=True)
def run_update(vault: Path) -> int:
"""运行更新检查与安装"""
local_cfg = vault / "paperforge.json"
if not local_cfg.exists():
_log("[ERR] 未找到 paperforge.json", "r")
return 1
local = json.loads(local_cfg.read_text(encoding="utf-8")).get("version", "unknown")
remote = _remote_version()
_log("=" * 50, "b")
_log("PaperForge Lite 更新", "b")
_log("=" * 50, "b")
_log(f"本地版本: {local}", "c")
_log(f"远程版本: {remote or 'unknown'}", "c")
if not remote:
_log("[ERR] 无法获取远程版本", "r")
return 1
try:
needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple(int(x) for x in local.split(".") if x.isdigit())
except ValueError:
needs = remote != local
if not needs:
_log("[OK] 当前已是最新版本", "g")
return 0
_log(f"\n[INFO] 发现新版本: {local} -> {remote}", "y")
_log("[WARN] 更新前建议备份 Vault", "y")
ans = input(_color("确认更新? [y/N]: ", "y")).strip().lower()
if ans not in ("y", "yes"):
_log("[INFO] 已取消", "c")
return 0
if (vault / ".git").is_dir():
success = update_via_git(vault)
else:
success = update_via_zip(vault)
if success:
_log("\n[OK] 更新完成!请重启 Obsidian", "g")
return 0 if success else 1
# =============================================================================
# Main
# =============================================================================

View file

@ -929,7 +929,11 @@ class DeployStep(StepScreen):
wizard_dir = Path(__file__).parent.resolve()
# 如果 wizard 在 github-release/ 下repo_root 就是 github-release/
# 如果 wizard 在 scripts/ 下repo_root 是父目录
if (wizard_dir / "pipeline").exists():
if (wizard_dir / "paperforge").exists():
repo_root = wizard_dir
elif (wizard_dir.parent / "paperforge").exists():
repo_root = wizard_dir.parent
elif (wizard_dir / "pipeline").exists():
repo_root = wizard_dir
elif (wizard_dir.parent / "pipeline").exists():
repo_root = wizard_dir.parent
@ -956,17 +960,31 @@ class DeployStep(StepScreen):
# 4. 复制脚本(从安装包到 Vault
import shutil
# Copy pipeline worker to PaperForge/worker/scripts/
worker_src = repo_root / "pipeline/worker/scripts/literature_pipeline.py"
worker_dst = pf_path / "worker/scripts/literature_pipeline.py"
# Copy worker modules to PaperForge/worker/scripts/
worker_src = repo_root / "paperforge/worker/sync.py"
worker_dst = pf_path / "worker/scripts/sync.py"
if worker_src.exists():
import shutil
shutil.copy2(worker_src, worker_dst)
# Also copy other worker modules
for mod in ["ocr.py", "repair.py", "status.py", "deep_reading.py", "update.py", "base_views.py", "__init__.py"]:
mod_src = repo_root / "paperforge/worker" / mod
if mod_src.exists():
shutil.copy2(mod_src, pf_path / "worker/scripts" / mod)
else:
self.set_status(f"错误:找不到 worker 脚本: {worker_src}", False)
return False
# Fallback to old pipeline location for backward compatibility
worker_src = repo_root / "pipeline/worker/scripts/literature_pipeline.py"
worker_dst = pf_path / "worker/scripts/literature_pipeline.py"
if worker_src.exists():
shutil.copy2(worker_src, worker_dst)
else:
self.set_status(f"错误:找不到 worker 脚本: {worker_src}", False)
return False
# Copy ld_deep.py
ld_src = repo_root / "skills/literature-qa/scripts/ld_deep.py"
# Copy ld_deep.py (prefer paperforge/skills, fallback to skills/)
ld_src = repo_root / "paperforge/skills/literature-qa/scripts/ld_deep.py"
if not ld_src.exists():
ld_src = repo_root / "skills/literature-qa/scripts/ld_deep.py"
ld_dst = vault / skill_dir / "literature-qa/scripts/ld_deep.py"
if ld_src.exists():
shutil.copy2(ld_src, ld_dst)
@ -974,8 +992,10 @@ class DeployStep(StepScreen):
self.set_status(f"错误:找不到 ld_deep.py: {ld_src}", False)
return False
# Copy subagent prompt
prompt_src = repo_root / "skills/literature-qa/prompt_deep_subagent.md"
# Copy subagent prompt (prefer paperforge/skills, fallback to skills/)
prompt_src = repo_root / "paperforge/skills/literature-qa/prompt_deep_subagent.md"
if not prompt_src.exists():
prompt_src = repo_root / "skills/literature-qa/prompt_deep_subagent.md"
prompt_dst = vault / skill_dir / "literature-qa/prompt_deep_subagent.md"
if prompt_src.exists():
shutil.copy2(prompt_src, prompt_dst)
@ -983,8 +1003,10 @@ class DeployStep(StepScreen):
self.set_status(f"错误:找不到 prompt_deep_subagent.md: {prompt_src}", False)
return False
# Copy chart-reading guides
chart_src = repo_root / "skills/literature-qa/chart-reading"
# Copy chart-reading guides (prefer paperforge/skills, fallback to skills/)
chart_src = repo_root / "paperforge/skills/literature-qa/chart-reading"
if not chart_src.exists():
chart_src = repo_root / "skills/literature-qa/chart-reading"
chart_dst = vault / skill_dir / "literature-qa/chart-reading"
if chart_src.exists() and chart_src.is_dir():
for f in chart_src.glob("*.md"):
@ -1144,7 +1166,7 @@ class DoneStep(StepScreen):
yield from super().compose()
vault_config = getattr(self.app, 'vault_config', {})
system_dir = vault_config.get('system_dir', '99_System')
worker_cmd = f"python {system_dir}/PaperForge/worker/scripts/literature_pipeline.py --vault ."
worker_cmd = f"python -m paperforge sync --vault ."
yield Markdown(f"""
## 安装完成!