mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 17:00:23 +00:00
feat(phase-12): extract worker modules from literature_pipeline.py
- Create paperforge/worker/ package with sync.py, ocr.py, repair.py, status.py, deep_reading.py, update.py, base_views.py - Update paperforge/commands/*.py to import from new worker modules - Update paperforge/cli.py _import_worker_functions for new paths - Update paperforge/config.py ld_deep_script resolution with fallback - Update setup_wizard.py for new worker/skills paths - Add function-level imports to break circular dependencies
This commit is contained in:
parent
dca498e85b
commit
498a9edfe5
16 changed files with 5329 additions and 30 deletions
|
|
@ -86,15 +86,12 @@ def _import_worker_functions() -> None:
|
|||
global run_status, run_selection_sync, run_index_refresh
|
||||
global run_deep_reading, run_repair, run_ocr, ensure_base_views
|
||||
|
||||
from pipeline.worker.scripts.literature_pipeline import (
|
||||
run_status as _rs,
|
||||
run_selection_sync as _rss,
|
||||
run_index_refresh as _rir,
|
||||
run_deep_reading as _rdr,
|
||||
run_repair as _rr,
|
||||
run_ocr as _ro,
|
||||
ensure_base_views as _ebu,
|
||||
)
|
||||
from paperforge.worker.status import run_status as _rs
|
||||
from paperforge.worker.sync import run_selection_sync as _rss, run_index_refresh as _rir
|
||||
from paperforge.worker.deep_reading import run_deep_reading as _rdr
|
||||
from paperforge.worker.repair import run_repair as _rr
|
||||
from paperforge.worker.ocr import run_ocr as _ro
|
||||
from paperforge.worker.base_views import ensure_base_views as _ebu
|
||||
|
||||
if run_status is None:
|
||||
run_status = _rs
|
||||
|
|
@ -337,7 +334,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
return 0
|
||||
|
||||
if args.command == "doctor":
|
||||
from pipeline.worker.scripts.literature_pipeline import run_doctor
|
||||
from paperforge.worker.status import run_doctor
|
||||
return run_doctor(vault)
|
||||
|
||||
print(f"Error: unknown command {args.command}", file=sys.stderr)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def _get_run_deep_reading():
|
|||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from pipeline.worker.scripts.literature_pipeline import run_deep_reading
|
||||
from paperforge.worker.deep_reading import run_deep_reading
|
||||
|
||||
return run_deep_reading
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ def _get_run_ocr():
|
|||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from pipeline.worker.scripts.literature_pipeline import run_ocr
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
return run_ocr
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def _get_run_repair():
|
|||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from pipeline.worker.scripts.literature_pipeline import run_repair
|
||||
from paperforge.worker.repair import run_repair
|
||||
|
||||
return run_repair
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def _get_run_status():
|
|||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from pipeline.worker.scripts.literature_pipeline import run_status
|
||||
from paperforge.worker.status import run_status
|
||||
|
||||
return run_status
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def _get_run_selection_sync():
|
|||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from pipeline.worker.scripts.literature_pipeline import run_selection_sync
|
||||
from paperforge.worker.sync import run_selection_sync
|
||||
|
||||
return run_selection_sync
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ def _get_run_index_refresh():
|
|||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from pipeline.worker.scripts.literature_pipeline import run_index_refresh
|
||||
from paperforge.worker.sync import run_index_refresh
|
||||
|
||||
return run_index_refresh
|
||||
|
||||
|
|
|
|||
|
|
@ -256,8 +256,17 @@ def paperforge_paths(
|
|||
# Resolve script paths relative to vault (for portability in copied installs)
|
||||
# worker_script: look relative to vault root
|
||||
worker_script = vault / "pipeline" / "worker" / "scripts" / "literature_pipeline.py"
|
||||
# ld_deep_script: look relative to skill_dir
|
||||
# ld_deep_script: look relative to skill_dir first, then repo paperforge/skills for dev
|
||||
ld_deep_script = skill_path / "literature-qa" / "scripts" / "ld_deep.py"
|
||||
if not ld_deep_script.exists():
|
||||
repo_skill = Path(__file__).parent / "skills" / "literature-qa" / "scripts" / "ld_deep.py"
|
||||
if repo_skill.exists():
|
||||
ld_deep_script = repo_skill
|
||||
else:
|
||||
# Backward compat: old skills/ location during transition
|
||||
old_repo_skill = Path(__file__).parent.parent / "skills" / "literature-qa" / "scripts" / "ld_deep.py"
|
||||
if old_repo_skill.exists():
|
||||
ld_deep_script = old_repo_skill
|
||||
|
||||
return {
|
||||
"vault": vault,
|
||||
|
|
|
|||
34
paperforge/worker/__init__.py
Normal file
34
paperforge/worker/__init__.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""PaperForge worker modules — migrated from pipeline/worker/scripts."""
|
||||
|
||||
from paperforge.worker.sync import (
|
||||
load_export_rows,
|
||||
run_selection_sync,
|
||||
run_index_refresh,
|
||||
obsidian_wikilink_for_pdf,
|
||||
absolutize_vault_path,
|
||||
_normalize_attachment_path,
|
||||
_identify_main_pdf,
|
||||
)
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
from paperforge.worker.repair import run_repair
|
||||
from paperforge.worker.status import run_doctor, run_status
|
||||
from paperforge.worker.deep_reading import run_deep_reading
|
||||
from paperforge.worker.base_views import build_base_views, merge_base_views, ensure_base_views
|
||||
|
||||
__all__ = [
|
||||
"load_export_rows",
|
||||
"run_selection_sync",
|
||||
"run_index_refresh",
|
||||
"run_ocr",
|
||||
"run_repair",
|
||||
"run_doctor",
|
||||
"run_status",
|
||||
"run_deep_reading",
|
||||
"build_base_views",
|
||||
"merge_base_views",
|
||||
"ensure_base_views",
|
||||
"obsidian_wikilink_for_pdf",
|
||||
"absolutize_vault_path",
|
||||
"_normalize_attachment_path",
|
||||
"_identify_main_pdf",
|
||||
]
|
||||
516
paperforge/worker/base_views.py
Normal file
516
paperforge/worker/base_views.py
Normal file
|
|
@ -0,0 +1,516 @@
|
|||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from json import JSONDecodeError
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
import requests
|
||||
import fitz
|
||||
from PIL import Image
|
||||
|
||||
STANDARD_VIEW_NAMES = frozenset([
|
||||
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
|
||||
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
|
||||
])
|
||||
|
||||
def load_simple_env(env_path: Path) -> None:
|
||||
if not env_path.exists():
|
||||
return
|
||||
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
key = key.strip()
|
||||
if not key or key in os.environ:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
def read_json(path: Path):
|
||||
return json.loads(path.read_text(encoding='utf-8'))
|
||||
_JOURNAL_DB: dict[str, dict] | None = None
|
||||
|
||||
def load_journal_db(vault: Path) -> dict[str, dict]:
|
||||
"""Load zoterostyle.json journal database."""
|
||||
global _JOURNAL_DB
|
||||
if _JOURNAL_DB is not None:
|
||||
return _JOURNAL_DB
|
||||
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
|
||||
if zoterostyle_path.exists():
|
||||
try:
|
||||
_JOURNAL_DB = read_json(zoterostyle_path)
|
||||
except (JSONDecodeError, Exception):
|
||||
_JOURNAL_DB = {}
|
||||
else:
|
||||
_JOURNAL_DB = {}
|
||||
return _JOURNAL_DB
|
||||
|
||||
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
|
||||
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
|
||||
if not journal_name:
|
||||
return ''
|
||||
journal_db = load_journal_db(vault)
|
||||
if journal_name in journal_db:
|
||||
rank_data = journal_db[journal_name].get('rank', {})
|
||||
if isinstance(rank_data, dict):
|
||||
sciif = rank_data.get('sciif', '')
|
||||
if sciif:
|
||||
return str(sciif)
|
||||
if extra:
|
||||
if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra)
|
||||
if if_match:
|
||||
return if_match.group(1)
|
||||
return ''
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.read_text(encoding='utf-8').splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
def write_jsonl(path: Path, rows) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
|
||||
if text:
|
||||
text += '\n'
|
||||
path.write_text(text, encoding='utf-8')
|
||||
|
||||
def yaml_quote(value: str) -> str:
|
||||
if isinstance(value, bool):
|
||||
return 'true' if value else 'false'
|
||||
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
def yaml_block(value: str) -> list[str]:
|
||||
value = (value or '').strip()
|
||||
if not value:
|
||||
return ['abstract: |-', ' ']
|
||||
lines = ['abstract: |-']
|
||||
for line in value.splitlines():
|
||||
lines.append(f' {line}')
|
||||
return lines
|
||||
|
||||
def yaml_list(key: str, values) -> list[str]:
|
||||
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
|
||||
if not cleaned:
|
||||
return [f'{key}: []']
|
||||
lines = [f'{key}:']
|
||||
for value in cleaned:
|
||||
lines.append(f' - {yaml_quote(value)}')
|
||||
return lines
|
||||
|
||||
def slugify_filename(text: str) -> str:
|
||||
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
|
||||
return cleaned[:120] or 'untitled'
|
||||
|
||||
def _extract_year(value: str) -> str:
|
||||
match = re.search('(19|20)\\d{2}', value or '')
|
||||
return match.group(0) if match else ''
|
||||
|
||||
|
||||
def load_vault_config(vault: Path) -> dict:
|
||||
"""Read vault configuration — delegates to shared resolver.
|
||||
|
||||
Preserves the public name for legacy callers. Configuration precedence:
|
||||
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
|
||||
"""
|
||||
from paperforge.config import load_vault_config as _shared_load_vault_config
|
||||
return _shared_load_vault_config(vault)
|
||||
|
||||
|
||||
def pipeline_paths(vault: Path) -> dict[str, Path]:
|
||||
"""Build complete PaperForge path inventory — delegates to shared resolver.
|
||||
|
||||
Returns paths from paperforge.config.paperforge_paths() plus
|
||||
worker-only keys. Preserves all legacy keys for existing callers.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths as _shared_paperforge_paths
|
||||
|
||||
shared = _shared_paperforge_paths(vault)
|
||||
|
||||
cfg = load_vault_config(vault)
|
||||
system_dir = cfg["system_dir"]
|
||||
resources_dir = cfg["resources_dir"]
|
||||
control_dir = cfg["control_dir"]
|
||||
|
||||
root = shared["paperforge"]
|
||||
control_root = shared["control"]
|
||||
|
||||
return {
|
||||
**shared,
|
||||
# Worker-only keys (added on top of shared resolver output)
|
||||
"pipeline": root,
|
||||
"candidates": root / "candidates" / "candidates.json",
|
||||
"candidate_inbox": root / "candidates" / "inbox",
|
||||
"candidate_archive": root / "candidates" / "archive",
|
||||
"search_tasks": root / "search" / "tasks",
|
||||
"search_archive": root / "search" / "archive",
|
||||
"search_results": root / "search" / "results",
|
||||
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
|
||||
"records": control_root / "candidate-records",
|
||||
"review": root / "candidates" / "review-latest.md",
|
||||
"config": root / "config" / "domain-collections.json",
|
||||
"queue": root / "writeback" / "writeback-queue.jsonl",
|
||||
"log": root / "writeback" / "writeback-log.jsonl",
|
||||
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
|
||||
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
|
||||
"index": root / "indexes" / "formal-library.json",
|
||||
"ocr_queue": root / "ocr" / "ocr-queue.json",
|
||||
}
|
||||
|
||||
def load_domain_config(paths: dict[str, Path]) -> dict:
|
||||
"""Load or create the Lite domain mapping from export JSON files."""
|
||||
config_path = paths['config']
|
||||
if config_path.exists():
|
||||
config = read_json(config_path)
|
||||
else:
|
||||
config = {"domains": []}
|
||||
domains = config.setdefault("domains", [])
|
||||
known_exports = {str(entry.get("export_file", "")) for entry in domains}
|
||||
changed = not config_path.exists()
|
||||
for export_path in sorted(paths['exports'].glob('*.json')):
|
||||
if export_path.name in known_exports:
|
||||
continue
|
||||
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
|
||||
known_exports.add(export_path.name)
|
||||
changed = True
|
||||
if changed:
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json(config_path, config)
|
||||
return config
|
||||
|
||||
def base_markdown_filter(path: Path, vault: Path) -> str:
|
||||
try:
|
||||
return str(path.relative_to(vault)).replace('\\', '/')
|
||||
except ValueError:
|
||||
return str(path).replace('\\', '/')
|
||||
|
||||
PAPERFORGE_VIEW_PREFIX = "# PAPERFORGE_VIEW: "
|
||||
|
||||
def build_base_views(domain: str) -> list[dict]:
|
||||
"""Build the 8-view list for a domain Base file.
|
||||
|
||||
Args:
|
||||
domain: The domain name (e.g., "骨科") — passed for compatibility but each view has fixed name/filter.
|
||||
|
||||
Returns:
|
||||
List of 8 view dicts, each with keys: name (str), order (list), filter (str|None).
|
||||
"""
|
||||
return [
|
||||
{
|
||||
"name": "控制面板",
|
||||
"order": ["file.name", "title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
|
||||
"filter": None,
|
||||
},
|
||||
{
|
||||
"name": "推荐分析",
|
||||
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
|
||||
"filter": "analyze = true AND recommend_analyze = true",
|
||||
},
|
||||
{
|
||||
"name": "待 OCR",
|
||||
"order": ["year", "title", "has_pdf", "do_ocr", "ocr_status", "pdf_path"],
|
||||
"filter": 'do_ocr = true AND ocr_status = "pending"',
|
||||
},
|
||||
{
|
||||
"name": "OCR 完成",
|
||||
"order": ["year", "title", "has_pdf", "do_ocr", "ocr_status", "pdf_path"],
|
||||
"filter": 'ocr_status = "done"',
|
||||
},
|
||||
{
|
||||
"name": "待深度阅读",
|
||||
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"],
|
||||
"filter": 'analyze = true AND ocr_status = "done" AND deep_reading_status = "pending"',
|
||||
},
|
||||
{
|
||||
"name": "深度阅读完成",
|
||||
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"],
|
||||
"filter": 'deep_reading_status = "done"',
|
||||
},
|
||||
{
|
||||
"name": "正式卡片",
|
||||
"order": ["title", "year", "has_pdf", "deep_reading_status", "pdf_path"],
|
||||
"filter": 'deep_reading_status = "done"',
|
||||
},
|
||||
{
|
||||
"name": "全记录",
|
||||
"order": ["title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
|
||||
"filter": None,
|
||||
},
|
||||
]
|
||||
|
||||
def substitute_config_placeholders(content: str, paths: dict[str, Path]) -> str:
|
||||
"""Replace ${SCREAMING_SNAKE_CASE} path placeholders with vault-relative paths.
|
||||
|
||||
Args:
|
||||
content: The Base file content string with ${PLACEHOLDER} tokens.
|
||||
paths: dict of path key -> Path objects (from paperforge_paths()).
|
||||
|
||||
Returns:
|
||||
Content with placeholders replaced by vault-relative paths.
|
||||
Unrecognized placeholders are left unchanged.
|
||||
"""
|
||||
substitutions = {
|
||||
"LIBRARY_RECORDS": paths.get("library_records"),
|
||||
"LITERATURE": paths.get("literature"),
|
||||
"CONTROL_DIR": paths.get("control"),
|
||||
}
|
||||
result = content
|
||||
for key, path in substitutions.items():
|
||||
if path is not None:
|
||||
vault = paths.get("vault")
|
||||
if vault is not None:
|
||||
try:
|
||||
rel = path.relative_to(vault)
|
||||
result = result.replace(f"${{{key}}}", str(rel).replace(chr(92), "/"))
|
||||
except ValueError:
|
||||
result = result.replace(f"${{{key}}}", str(path).replace(chr(92), "/"))
|
||||
return result
|
||||
|
||||
|
||||
def _render_views_section(views: list[dict]) -> str:
|
||||
"""Render a list of view dicts to YAML views: section."""
|
||||
lines = []
|
||||
for v in views:
|
||||
lines.append(f" - type: table")
|
||||
lines.append(f' name: "{v["name"]}"')
|
||||
lines.append(f" order:")
|
||||
for col in v["order"]:
|
||||
lines.append(f" - {col}")
|
||||
if v["filter"]:
|
||||
lines.append(f' filter: \'{v["filter"]}\'')
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def merge_base_views(existing_content: str | None, new_views: list[dict]) -> str:
|
||||
"""Incrementally merge standard PaperForge views into an existing .base file.
|
||||
|
||||
Strategy:
|
||||
- PaperForge generates exactly 8 views with known names (STANDARD_VIEW_NAMES).
|
||||
- Any OTHER views in the existing file are user-defined and MUST be preserved.
|
||||
- Each PaperForge view is preceded by a PAPERFORGE_VIEW_PREFIX comment marker.
|
||||
- On refresh: replace ALL PaperForge views (identified by prefix) with fresh ones.
|
||||
- User views (no prefix) are left completely untouched.
|
||||
|
||||
Args:
|
||||
existing_content: Raw text of existing .base file (or None/empty for fresh generation).
|
||||
new_views: List of 8 view dicts from build_base_views().
|
||||
|
||||
Returns:
|
||||
Merged .base file content with PaperForge views updated, user views preserved.
|
||||
"""
|
||||
PROPERTIES_YAML = """properties:
|
||||
zotero_key:
|
||||
displayName: "Zotero Key"
|
||||
title:
|
||||
displayName: "Title"
|
||||
year:
|
||||
displayName: "Year"
|
||||
has_pdf:
|
||||
displayName: "PDF"
|
||||
do_ocr:
|
||||
displayName: "OCR"
|
||||
analyze:
|
||||
displayName: "Analyze"
|
||||
ocr_status:
|
||||
displayName: "OCR Status"
|
||||
deep_reading_status:
|
||||
displayName: "Deep Reading"
|
||||
pdf_path:
|
||||
displayName: "PDF Path"
|
||||
fulltext_md_path:
|
||||
displayName: "Fulltext"
|
||||
"""
|
||||
|
||||
if not existing_content or not existing_content.strip():
|
||||
fresh_views_yaml = _render_views_section(new_views)
|
||||
return f"""filters:
|
||||
and:
|
||||
- file.inFolder("{new_views[0]['name']}")
|
||||
{PROPERTIES_YAML}
|
||||
views:
|
||||
{fresh_views_yaml}"""
|
||||
|
||||
lines = existing_content.split("\n")
|
||||
views_start_idx = None
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip().startswith("views:"):
|
||||
views_start_idx = i
|
||||
break
|
||||
|
||||
if views_start_idx is None:
|
||||
fresh_views_yaml = _render_views_section(new_views)
|
||||
return f"""{existing_content}
|
||||
|
||||
# --- PaperForge views regenerated (views: section was missing) ---
|
||||
{PROPERTIES_YAML}
|
||||
views:
|
||||
{fresh_views_yaml}"""
|
||||
|
||||
header_lines = lines[:views_start_idx + 1]
|
||||
|
||||
new_pf_blocks = []
|
||||
for v in new_views:
|
||||
rendered = f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n"
|
||||
rendered += f" - type: table\n"
|
||||
rendered += f' name: "{v["name"]}"\n'
|
||||
rendered += f" order:\n"
|
||||
for col in v["order"]:
|
||||
rendered += f" - {col}\n"
|
||||
if v["filter"]:
|
||||
rendered += f' filter: \'{v["filter"]}\'\n'
|
||||
else:
|
||||
rendered += '\n'
|
||||
new_pf_blocks.append((v["name"], rendered))
|
||||
|
||||
rebuilt_views_lines = []
|
||||
pf_names_seen = set()
|
||||
i = views_start_idx + 1
|
||||
pending_pf_view_name = None
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
|
||||
if line.startswith(PAPERFORGE_VIEW_PREFIX):
|
||||
pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX):].strip()
|
||||
pf_names_seen.add(pending_pf_view_name)
|
||||
i += 1
|
||||
continue
|
||||
elif pending_pf_view_name is not None and line.strip().startswith("- type: table"):
|
||||
pf_block = next((b for n, b in new_pf_blocks if n == pending_pf_view_name), None)
|
||||
if pf_block:
|
||||
rebuilt_views_lines.append(pf_block)
|
||||
pending_pf_view_name = None
|
||||
i += 1
|
||||
continue
|
||||
elif line.strip().startswith("- type: table"):
|
||||
pending_pf_view_name = None
|
||||
view_block_lines = [line]
|
||||
i += 1
|
||||
while i < len(lines):
|
||||
next_line = lines[i]
|
||||
if next_line.strip().startswith(PAPERFORGE_VIEW_PREFIX):
|
||||
break
|
||||
if next_line.strip().startswith("- type: table"):
|
||||
break
|
||||
if next_line.strip() and not next_line.startswith(" ") and not next_line.startswith("\t"):
|
||||
break
|
||||
view_block_lines.append(next_line)
|
||||
i += 1
|
||||
rebuilt_views_lines.append("\n".join(view_block_lines))
|
||||
continue
|
||||
else:
|
||||
pending_pf_view_name = None
|
||||
i += 1
|
||||
|
||||
for view_name, pf_block in new_pf_blocks:
|
||||
if view_name not in pf_names_seen:
|
||||
rebuilt_views_lines.append(pf_block)
|
||||
|
||||
result_lines = header_lines + rebuilt_views_lines
|
||||
return "\n".join(result_lines)
|
||||
|
||||
|
||||
def _build_base_yaml(folder_filter: str, views: list[dict]) -> str:
|
||||
"""Build complete .base YAML with PAPERFORGE_VIEW_PREFIX markers on each view."""
|
||||
views_yaml = ""
|
||||
for v in views:
|
||||
views_yaml += f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n"
|
||||
views_yaml += f" - type: table\n"
|
||||
views_yaml += f' name: "{v["name"]}"\n'
|
||||
views_yaml += f" order:\n"
|
||||
for col in v["order"]:
|
||||
views_yaml += f" - {col}\n"
|
||||
if v["filter"]:
|
||||
views_yaml += f' filter: \'{v["filter"]}\'\n'
|
||||
else:
|
||||
views_yaml += '\n'
|
||||
views_yaml = views_yaml.rstrip('\n')
|
||||
return f"""filters:
|
||||
and:
|
||||
- file.inFolder("{folder_filter}")
|
||||
properties:
|
||||
zotero_key:
|
||||
displayName: "Zotero Key"
|
||||
title:
|
||||
displayName: "Title"
|
||||
year:
|
||||
displayName: "Year"
|
||||
has_pdf:
|
||||
displayName: "PDF"
|
||||
do_ocr:
|
||||
displayName: "OCR"
|
||||
analyze:
|
||||
displayName: "Analyze"
|
||||
ocr_status:
|
||||
displayName: "OCR Status"
|
||||
deep_reading_status:
|
||||
displayName: "Deep Reading"
|
||||
pdf_path:
|
||||
displayName: "PDF Path"
|
||||
fulltext_md_path:
|
||||
displayName: "Fulltext"
|
||||
views:
|
||||
{views_yaml}"""
|
||||
|
||||
|
||||
def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: bool = False) -> None:
|
||||
"""Generate/refresh Domain Base files with incremental merge (preserves user views).
|
||||
|
||||
Each PaperForge standard view is marked with PAPERFORGE_VIEW_PREFIX. On refresh,
|
||||
only PaperForge views are replaced; user-defined views are preserved.
|
||||
"""
|
||||
paths["bases"].mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def refresh_base(base_path: Path, folder_filter: str, views: list[dict]) -> None:
|
||||
"""Refresh a single .base file: merge PaperForge views, preserve user views."""
|
||||
if base_path.exists() and not force:
|
||||
existing = base_path.read_text(encoding="utf-8")
|
||||
merged = merge_base_views(existing, views)
|
||||
else:
|
||||
merged = _build_base_yaml(folder_filter, views)
|
||||
merged = substitute_config_placeholders(merged, paths)
|
||||
base_path.write_text(merged, encoding="utf-8")
|
||||
|
||||
seen_domains = set()
|
||||
for entry in config.get("domains", []):
|
||||
domain = str(entry.get("domain", "") or "").strip()
|
||||
if not domain or domain in seen_domains:
|
||||
continue
|
||||
seen_domains.add(domain)
|
||||
|
||||
domain_views = build_base_views(domain)
|
||||
folder_filter = f"${{LIBRARY_RECORDS}}/{domain}"
|
||||
base_path = paths["bases"] / f"{slugify_filename(domain)}.base"
|
||||
refresh_base(base_path, folder_filter, domain_views)
|
||||
|
||||
hub_views = build_base_views("Literature Hub")
|
||||
hub_path = paths["bases"] / "Literature Hub.base"
|
||||
refresh_base(hub_path, "${LIBRARY_RECORDS}", hub_views)
|
||||
|
||||
all_views = build_base_views("All Records")
|
||||
pf_base = paths["bases"] / "PaperForge.base"
|
||||
refresh_base(pf_base, "${LIBRARY_RECORDS}", all_views)
|
||||
|
||||
|
||||
324
paperforge/worker/deep_reading.py
Normal file
324
paperforge/worker/deep_reading.py
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from json import JSONDecodeError
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
import requests
|
||||
import fitz
|
||||
from PIL import Image
|
||||
|
||||
from paperforge.worker.sync import has_deep_reading_content, load_export_rows
|
||||
from paperforge.worker.base_views import ensure_base_views
|
||||
from paperforge.worker.ocr import validate_ocr_meta
|
||||
|
||||
STANDARD_VIEW_NAMES = frozenset([
|
||||
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
|
||||
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
|
||||
])
|
||||
|
||||
def load_simple_env(env_path: Path) -> None:
|
||||
if not env_path.exists():
|
||||
return
|
||||
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
key = key.strip()
|
||||
if not key or key in os.environ:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
def read_json(path: Path):
|
||||
return json.loads(path.read_text(encoding='utf-8'))
|
||||
_JOURNAL_DB: dict[str, dict] | None = None
|
||||
|
||||
def load_journal_db(vault: Path) -> dict[str, dict]:
|
||||
"""Load zoterostyle.json journal database."""
|
||||
global _JOURNAL_DB
|
||||
if _JOURNAL_DB is not None:
|
||||
return _JOURNAL_DB
|
||||
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
|
||||
if zoterostyle_path.exists():
|
||||
try:
|
||||
_JOURNAL_DB = read_json(zoterostyle_path)
|
||||
except (JSONDecodeError, Exception):
|
||||
_JOURNAL_DB = {}
|
||||
else:
|
||||
_JOURNAL_DB = {}
|
||||
return _JOURNAL_DB
|
||||
|
||||
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
|
||||
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
|
||||
if not journal_name:
|
||||
return ''
|
||||
journal_db = load_journal_db(vault)
|
||||
if journal_name in journal_db:
|
||||
rank_data = journal_db[journal_name].get('rank', {})
|
||||
if isinstance(rank_data, dict):
|
||||
sciif = rank_data.get('sciif', '')
|
||||
if sciif:
|
||||
return str(sciif)
|
||||
if extra:
|
||||
if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra)
|
||||
if if_match:
|
||||
return if_match.group(1)
|
||||
return ''
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.read_text(encoding='utf-8').splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
def write_jsonl(path: Path, rows) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
|
||||
if text:
|
||||
text += '\n'
|
||||
path.write_text(text, encoding='utf-8')
|
||||
|
||||
def yaml_quote(value: str) -> str:
|
||||
if isinstance(value, bool):
|
||||
return 'true' if value else 'false'
|
||||
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
def yaml_block(value: str) -> list[str]:
|
||||
value = (value or '').strip()
|
||||
if not value:
|
||||
return ['abstract: |-', ' ']
|
||||
lines = ['abstract: |-']
|
||||
for line in value.splitlines():
|
||||
lines.append(f' {line}')
|
||||
return lines
|
||||
|
||||
def yaml_list(key: str, values) -> list[str]:
|
||||
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
|
||||
if not cleaned:
|
||||
return [f'{key}: []']
|
||||
lines = [f'{key}:']
|
||||
for value in cleaned:
|
||||
lines.append(f' - {yaml_quote(value)}')
|
||||
return lines
|
||||
|
||||
def slugify_filename(text: str) -> str:
|
||||
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
|
||||
return cleaned[:120] or 'untitled'
|
||||
|
||||
def _extract_year(value: str) -> str:
|
||||
match = re.search('(19|20)\\d{2}', value or '')
|
||||
return match.group(0) if match else ''
|
||||
|
||||
|
||||
def load_vault_config(vault: Path) -> dict:
|
||||
"""Read vault configuration — delegates to shared resolver.
|
||||
|
||||
Preserves the public name for legacy callers. Configuration precedence:
|
||||
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
|
||||
"""
|
||||
from paperforge.config import load_vault_config as _shared_load_vault_config
|
||||
return _shared_load_vault_config(vault)
|
||||
|
||||
|
||||
def pipeline_paths(vault: Path) -> dict[str, Path]:
|
||||
"""Build complete PaperForge path inventory — delegates to shared resolver.
|
||||
|
||||
Returns paths from paperforge.config.paperforge_paths() plus
|
||||
worker-only keys. Preserves all legacy keys for existing callers.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths as _shared_paperforge_paths
|
||||
|
||||
shared = _shared_paperforge_paths(vault)
|
||||
|
||||
cfg = load_vault_config(vault)
|
||||
system_dir = cfg["system_dir"]
|
||||
resources_dir = cfg["resources_dir"]
|
||||
control_dir = cfg["control_dir"]
|
||||
|
||||
root = shared["paperforge"]
|
||||
control_root = shared["control"]
|
||||
|
||||
return {
|
||||
**shared,
|
||||
# Worker-only keys (added on top of shared resolver output)
|
||||
"pipeline": root,
|
||||
"candidates": root / "candidates" / "candidates.json",
|
||||
"candidate_inbox": root / "candidates" / "inbox",
|
||||
"candidate_archive": root / "candidates" / "archive",
|
||||
"search_tasks": root / "search" / "tasks",
|
||||
"search_archive": root / "search" / "archive",
|
||||
"search_results": root / "search" / "results",
|
||||
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
|
||||
"records": control_root / "candidate-records",
|
||||
"review": root / "candidates" / "review-latest.md",
|
||||
"config": root / "config" / "domain-collections.json",
|
||||
"queue": root / "writeback" / "writeback-queue.jsonl",
|
||||
"log": root / "writeback" / "writeback-log.jsonl",
|
||||
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
|
||||
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
|
||||
"index": root / "indexes" / "formal-library.json",
|
||||
"ocr_queue": root / "ocr" / "ocr-queue.json",
|
||||
}
|
||||
|
||||
def load_domain_config(paths: dict[str, Path]) -> dict:
|
||||
"""Load or create the Lite domain mapping from export JSON files."""
|
||||
config_path = paths['config']
|
||||
if config_path.exists():
|
||||
config = read_json(config_path)
|
||||
else:
|
||||
config = {"domains": []}
|
||||
domains = config.setdefault("domains", [])
|
||||
known_exports = {str(entry.get("export_file", "")) for entry in domains}
|
||||
changed = not config_path.exists()
|
||||
for export_path in sorted(paths['exports'].glob('*.json')):
|
||||
if export_path.name in known_exports:
|
||||
continue
|
||||
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
|
||||
known_exports.add(export_path.name)
|
||||
changed = True
|
||||
if changed:
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json(config_path, config)
|
||||
return config
|
||||
|
||||
def _resolve_formal_note_path(vault: Path, zotero_key: str, domain: str) -> Path | None:
|
||||
"""Resolve formal literature note by zotero_key."""
|
||||
lit_root = pipeline_paths(vault)['literature']
|
||||
domain_dir = lit_root / domain
|
||||
if not domain_dir.exists():
|
||||
return None
|
||||
frontmatter_pattern = re.compile(f'^\\s*zotero_key:\\s*"?{re.escape(zotero_key)}"?\\s*$', re.MULTILINE)
|
||||
for note_path in domain_dir.rglob('*.md'):
|
||||
try:
|
||||
text = note_path.read_text(encoding='utf-8')
|
||||
except UnicodeDecodeError:
|
||||
text = note_path.read_text(encoding='utf-8', errors='ignore')
|
||||
if frontmatter_pattern.search(text):
|
||||
return note_path
|
||||
return None
|
||||
|
||||
def run_deep_reading(vault: Path, verbose: bool = False) -> int:
|
||||
"""Sync deep-reading status between formal notes and library records.
|
||||
|
||||
This worker does NOT generate content. It only:
|
||||
1. Scans formal literature notes for `## 🔍 精读` content
|
||||
2. Updates library-records/*.md frontmatter to match actual state
|
||||
3. Reports the queue of papers awaiting deep reading
|
||||
|
||||
Actual content filling is done via /pf-deep (agent-driven).
|
||||
"""
|
||||
paths = pipeline_paths(vault)
|
||||
config = load_domain_config(paths)
|
||||
ensure_base_views(vault, paths, config)
|
||||
domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']}
|
||||
synced = 0
|
||||
pending_queue: list[dict] = []
|
||||
for export_path in sorted(paths['exports'].glob('*.json')):
|
||||
domain = domain_lookup.get(export_path.name, export_path.stem)
|
||||
for item in load_export_rows(export_path):
|
||||
key = item['key']
|
||||
record_dir = paths['library_records'] / domain
|
||||
record_path = record_dir / f'{key}.md'
|
||||
if not record_path.exists():
|
||||
continue
|
||||
record_text = record_path.read_text(encoding='utf-8')
|
||||
analyze_match = re.search('^analyze:\\s*(true|false)$', record_text, re.MULTILINE)
|
||||
is_analyze = analyze_match and analyze_match.group(1) == 'true'
|
||||
do_ocr_match = re.search('^do_ocr:\\s*(true|false)$', record_text, re.MULTILINE)
|
||||
is_do_ocr = do_ocr_match and do_ocr_match.group(1) == 'true'
|
||||
note_path = _resolve_formal_note_path(vault, key, domain)
|
||||
has_content = False
|
||||
if note_path and note_path.exists():
|
||||
note_text = note_path.read_text(encoding='utf-8')
|
||||
has_content = has_deep_reading_content(note_text)
|
||||
correct_status = 'done' if has_content else 'pending'
|
||||
status_match = re.search('^deep_reading_status:\\s*"??"?$', record_text, re.MULTILINE)
|
||||
current_status = status_match.group(1) if status_match else 'pending'
|
||||
if current_status != correct_status:
|
||||
new_text = re.sub('^deep_reading_status:\\s*"?.*?"?$', f'deep_reading_status: {yaml_quote(correct_status)}', record_text, flags=re.MULTILINE, count=1)
|
||||
record_path.write_text(new_text, encoding='utf-8')
|
||||
synced += 1
|
||||
if is_analyze and correct_status == 'pending':
|
||||
meta_path = paths['ocr'] / key / 'meta.json'
|
||||
ocr_status = 'pending'
|
||||
if meta_path.exists():
|
||||
try:
|
||||
meta = read_json(meta_path)
|
||||
validated_status, error_msg = validate_ocr_meta(paths, meta)
|
||||
ocr_status = validated_status
|
||||
except Exception:
|
||||
pass
|
||||
pending_queue.append({
|
||||
'zotero_key': key,
|
||||
'domain': domain,
|
||||
'title': item.get('title', ''),
|
||||
'ocr_status': ocr_status,
|
||||
'is_analyze': is_analyze,
|
||||
'is_do_ocr': is_do_ocr,
|
||||
})
|
||||
if pending_queue:
|
||||
ready = [q for q in pending_queue if q['ocr_status'] == 'done']
|
||||
waiting = [q for q in pending_queue if q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing')]
|
||||
blocked = [q for q in pending_queue if q['is_analyze'] and q['ocr_status'] not in ('done', '') and not (q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing'))]
|
||||
report_lines = ['# 待精读队列', '']
|
||||
if ready:
|
||||
report_lines.extend([f'## 就绪 ({len(ready)} 篇) — OCR 已完成,可直接 /pf-deep', ''])
|
||||
for q in ready:
|
||||
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']}")
|
||||
report_lines.append('')
|
||||
if waiting:
|
||||
report_lines.extend([f'## 等待 OCR ({len(waiting)} 篇)', ''])
|
||||
for q in waiting:
|
||||
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status']}")
|
||||
report_lines.append('')
|
||||
if blocked:
|
||||
report_lines.extend([f'## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR', ''])
|
||||
for q in blocked:
|
||||
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}")
|
||||
report_lines.append('')
|
||||
if verbose:
|
||||
report_lines.append('### 修复步骤\n')
|
||||
for q in blocked:
|
||||
ocr_s = q['ocr_status'] or ''
|
||||
if not ocr_s or ocr_s == 'pending':
|
||||
fix = f"paperforge ocr"
|
||||
report_lines.append(f"- `{q['zotero_key']}`: 运行 `{fix}` 启动 OCR")
|
||||
elif ocr_s == 'processing':
|
||||
report_lines.append(f"- `{q['zotero_key']}`: OCR 进行中,请等待完成")
|
||||
elif ocr_s == 'failed':
|
||||
report_lines.append(f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息,然后重新运行 `paperforge ocr`")
|
||||
else:
|
||||
report_lines.append(f"- `{q['zotero_key']}`: 运行 `paperforge ocr` 重试")
|
||||
report_lines.append('')
|
||||
report_lines.extend(['## 操作', '', '- 对就绪论文,使用 `/pf-deep <zotero_key>` 触发精读', '- 批量触发:提供多个 key,用 subagent 并行处理', ''])
|
||||
else:
|
||||
report_lines = ['# 待精读队列', '', '所有 analyze=true 的论文已完成精读。', '']
|
||||
report_path = paths['pipeline'] / 'deep-reading-queue.md'
|
||||
report_path.write_text('\n'.join(report_lines), encoding='utf-8')
|
||||
print(f'deep-reading: synced {synced} records, {len(pending_queue)} pending')
|
||||
return 0
|
||||
|
||||
1376
paperforge/worker/ocr.py
Normal file
1376
paperforge/worker/ocr.py
Normal file
File diff suppressed because it is too large
Load diff
548
paperforge/worker/repair.py
Normal file
548
paperforge/worker/repair.py
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from json import JSONDecodeError
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
import requests
|
||||
import fitz
|
||||
from PIL import Image
|
||||
|
||||
from paperforge.worker.sync import (
|
||||
load_export_rows,
|
||||
obsidian_wikilink_for_pdf,
|
||||
update_frontmatter_field,
|
||||
)
|
||||
from paperforge.worker.deep_reading import _resolve_formal_note_path
|
||||
from paperforge.worker.ocr import validate_ocr_meta
|
||||
|
||||
STANDARD_VIEW_NAMES = frozenset([
|
||||
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
|
||||
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
|
||||
])
|
||||
|
||||
def load_simple_env(env_path: Path) -> None:
|
||||
if not env_path.exists():
|
||||
return
|
||||
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
key = key.strip()
|
||||
if not key or key in os.environ:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
def read_json(path: Path):
|
||||
return json.loads(path.read_text(encoding='utf-8'))
|
||||
_JOURNAL_DB: dict[str, dict] | None = None
|
||||
|
||||
def load_journal_db(vault: Path) -> dict[str, dict]:
|
||||
"""Load zoterostyle.json journal database."""
|
||||
global _JOURNAL_DB
|
||||
if _JOURNAL_DB is not None:
|
||||
return _JOURNAL_DB
|
||||
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
|
||||
if zoterostyle_path.exists():
|
||||
try:
|
||||
_JOURNAL_DB = read_json(zoterostyle_path)
|
||||
except (JSONDecodeError, Exception):
|
||||
_JOURNAL_DB = {}
|
||||
else:
|
||||
_JOURNAL_DB = {}
|
||||
return _JOURNAL_DB
|
||||
|
||||
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
|
||||
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
|
||||
if not journal_name:
|
||||
return ''
|
||||
journal_db = load_journal_db(vault)
|
||||
if journal_name in journal_db:
|
||||
rank_data = journal_db[journal_name].get('rank', {})
|
||||
if isinstance(rank_data, dict):
|
||||
sciif = rank_data.get('sciif', '')
|
||||
if sciif:
|
||||
return str(sciif)
|
||||
if extra:
|
||||
if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra)
|
||||
if if_match:
|
||||
return if_match.group(1)
|
||||
return ''
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.read_text(encoding='utf-8').splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
def write_jsonl(path: Path, rows) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
|
||||
if text:
|
||||
text += '\n'
|
||||
path.write_text(text, encoding='utf-8')
|
||||
|
||||
def yaml_quote(value: str) -> str:
|
||||
if isinstance(value, bool):
|
||||
return 'true' if value else 'false'
|
||||
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
def yaml_block(value: str) -> list[str]:
|
||||
value = (value or '').strip()
|
||||
if not value:
|
||||
return ['abstract: |-', ' ']
|
||||
lines = ['abstract: |-']
|
||||
for line in value.splitlines():
|
||||
lines.append(f' {line}')
|
||||
return lines
|
||||
|
||||
def yaml_list(key: str, values) -> list[str]:
|
||||
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
|
||||
if not cleaned:
|
||||
return [f'{key}: []']
|
||||
lines = [f'{key}:']
|
||||
for value in cleaned:
|
||||
lines.append(f' - {yaml_quote(value)}')
|
||||
return lines
|
||||
|
||||
def slugify_filename(text: str) -> str:
|
||||
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
|
||||
return cleaned[:120] or 'untitled'
|
||||
|
||||
def _extract_year(value: str) -> str:
|
||||
match = re.search('(19|20)\\d{2}', value or '')
|
||||
return match.group(0) if match else ''
|
||||
|
||||
|
||||
def load_vault_config(vault: Path) -> dict:
|
||||
"""Read vault configuration — delegates to shared resolver.
|
||||
|
||||
Preserves the public name for legacy callers. Configuration precedence:
|
||||
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
|
||||
"""
|
||||
from paperforge.config import load_vault_config as _shared_load_vault_config
|
||||
return _shared_load_vault_config(vault)
|
||||
|
||||
|
||||
def pipeline_paths(vault: Path) -> dict[str, Path]:
|
||||
"""Build complete PaperForge path inventory — delegates to shared resolver.
|
||||
|
||||
Returns paths from paperforge.config.paperforge_paths() plus
|
||||
worker-only keys. Preserves all legacy keys for existing callers.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths as _shared_paperforge_paths
|
||||
|
||||
shared = _shared_paperforge_paths(vault)
|
||||
|
||||
cfg = load_vault_config(vault)
|
||||
system_dir = cfg["system_dir"]
|
||||
resources_dir = cfg["resources_dir"]
|
||||
control_dir = cfg["control_dir"]
|
||||
|
||||
root = shared["paperforge"]
|
||||
control_root = shared["control"]
|
||||
|
||||
return {
|
||||
**shared,
|
||||
# Worker-only keys (added on top of shared resolver output)
|
||||
"pipeline": root,
|
||||
"candidates": root / "candidates" / "candidates.json",
|
||||
"candidate_inbox": root / "candidates" / "inbox",
|
||||
"candidate_archive": root / "candidates" / "archive",
|
||||
"search_tasks": root / "search" / "tasks",
|
||||
"search_archive": root / "search" / "archive",
|
||||
"search_results": root / "search" / "results",
|
||||
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
|
||||
"records": control_root / "candidate-records",
|
||||
"review": root / "candidates" / "review-latest.md",
|
||||
"config": root / "config" / "domain-collections.json",
|
||||
"queue": root / "writeback" / "writeback-queue.jsonl",
|
||||
"log": root / "writeback" / "writeback-log.jsonl",
|
||||
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
|
||||
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
|
||||
"index": root / "indexes" / "formal-library.json",
|
||||
"ocr_queue": root / "ocr" / "ocr-queue.json",
|
||||
}
|
||||
|
||||
def load_domain_config(paths: dict[str, Path]) -> dict:
|
||||
"""Load or create the Lite domain mapping from export JSON files."""
|
||||
config_path = paths['config']
|
||||
if config_path.exists():
|
||||
config = read_json(config_path)
|
||||
else:
|
||||
config = {"domains": []}
|
||||
domains = config.setdefault("domains", [])
|
||||
known_exports = {str(entry.get("export_file", "")) for entry in domains}
|
||||
changed = not config_path.exists()
|
||||
for export_path in sorted(paths['exports'].glob('*.json')):
|
||||
if export_path.name in known_exports:
|
||||
continue
|
||||
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
|
||||
known_exports.add(export_path.name)
|
||||
changed = True
|
||||
if changed:
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json(config_path, config)
|
||||
return config
|
||||
|
||||
def _find_export_for_domain(paths: dict[str, Path], domain: str) -> Path | None:
|
||||
"""Find the BBT export JSON file for a given domain."""
|
||||
for export_path in sorted(paths["exports"].glob("*.json")):
|
||||
if export_path.stem == domain:
|
||||
return export_path
|
||||
return None
|
||||
|
||||
|
||||
def _detect_path_errors(paths: dict[str, Path], verbose: bool = False) -> dict:
|
||||
"""Scan library-records for path_error fields.
|
||||
|
||||
Returns dict with:
|
||||
- total: total count of records with path_error
|
||||
- by_type: dict mapping error type -> count
|
||||
- records: list of record dicts with keys: path, zotero_key, domain, path_error, bbt_path_raw
|
||||
"""
|
||||
result: dict = {"total": 0, "by_type": {}, "records": []}
|
||||
if not paths["library_records"].exists():
|
||||
return result
|
||||
for record_path in paths["library_records"].rglob("*.md"):
|
||||
try:
|
||||
text = record_path.read_text(encoding="utf-8")
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
print(f"[repair] error reading {record_path}: {e}")
|
||||
continue
|
||||
err_match = re.search(r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE)
|
||||
if not err_match:
|
||||
continue
|
||||
path_error = err_match.group(1).strip()
|
||||
if not path_error:
|
||||
continue
|
||||
key_match = re.search(r'^zotero_key:\s*"?(.+?)"?\s*$', text, re.MULTILINE)
|
||||
if not key_match:
|
||||
continue
|
||||
zotero_key = key_match.group(1).strip()
|
||||
domain = record_path.parent.name
|
||||
bbt_match = re.search(r'^bbt_path_raw:\s*"(.*?)"\s*$', text, re.MULTILINE)
|
||||
bbt_path_raw = bbt_match.group(1) if bbt_match else ""
|
||||
result["total"] += 1
|
||||
result["by_type"][path_error] = result["by_type"].get(path_error, 0) + 1
|
||||
result["records"].append(
|
||||
{
|
||||
"path": record_path,
|
||||
"zotero_key": zotero_key,
|
||||
"domain": domain,
|
||||
"path_error": path_error,
|
||||
"bbt_path_raw": bbt_path_raw,
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def repair_pdf_paths(
|
||||
vault: Path,
|
||||
paths: dict[str, Path],
|
||||
error_records: list[dict],
|
||||
verbose: bool = False,
|
||||
) -> int:
|
||||
"""Re-resolve PDF paths for items with path_error.
|
||||
|
||||
Returns number of paths successfully fixed.
|
||||
"""
|
||||
fixed = 0
|
||||
from paperforge.pdf_resolver import resolve_pdf_path
|
||||
|
||||
cfg = load_vault_config(vault)
|
||||
zotero_dir = vault / cfg.get("system_dir", "99_System") / "Zotero"
|
||||
|
||||
# Cache export rows by domain to avoid reloading
|
||||
domain_exports: dict[str, list[dict]] = {}
|
||||
|
||||
for record in error_records:
|
||||
record_path = record["path"]
|
||||
zotero_key = record["zotero_key"]
|
||||
domain = record["domain"]
|
||||
path_error = record["path_error"]
|
||||
text = record["text"]
|
||||
|
||||
# For not_found errors, try to find the item in BBT export and re-process
|
||||
if path_error == "not_found":
|
||||
export_rows = domain_exports.get(domain)
|
||||
if export_rows is None:
|
||||
export_path = _find_export_for_domain(paths, domain)
|
||||
if export_path and export_path.exists():
|
||||
try:
|
||||
export_rows = load_export_rows(export_path)
|
||||
domain_exports[domain] = export_rows
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
print(
|
||||
f"[repair] error loading export for {domain}: {e}"
|
||||
)
|
||||
export_rows = []
|
||||
else:
|
||||
export_rows = []
|
||||
domain_exports[domain] = export_rows
|
||||
|
||||
item = next((r for r in export_rows if r["key"] == zotero_key), None)
|
||||
if item:
|
||||
pdf_path = item.get("pdf_path", "")
|
||||
if pdf_path:
|
||||
new_wikilink = obsidian_wikilink_for_pdf(
|
||||
pdf_path, vault, zotero_dir
|
||||
)
|
||||
if new_wikilink:
|
||||
new_text = update_frontmatter_field(
|
||||
text, "pdf_path", new_wikilink
|
||||
)
|
||||
new_text = update_frontmatter_field(
|
||||
new_text, "path_error", ""
|
||||
)
|
||||
new_text = update_frontmatter_field(
|
||||
new_text,
|
||||
"bbt_path_raw",
|
||||
item.get("bbt_path_raw", ""),
|
||||
)
|
||||
new_text = update_frontmatter_field(
|
||||
new_text,
|
||||
"zotero_storage_key",
|
||||
item.get("zotero_storage_key", ""),
|
||||
)
|
||||
if new_text != text:
|
||||
record_path.write_text(new_text, encoding="utf-8")
|
||||
fixed += 1
|
||||
if verbose:
|
||||
print(
|
||||
f"[repair] fixed path for {zotero_key}: {new_wikilink}"
|
||||
)
|
||||
continue
|
||||
|
||||
# For all errors, try resolving the current pdf_path
|
||||
pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE)
|
||||
if pdf_match:
|
||||
current_pdf = pdf_match.group(1).strip()
|
||||
if current_pdf:
|
||||
raw_path = current_pdf.strip("[]")
|
||||
resolved = resolve_pdf_path(raw_path, True, vault, zotero_dir)
|
||||
if resolved:
|
||||
new_text = update_frontmatter_field(text, "path_error", "")
|
||||
if new_text != text:
|
||||
record_path.write_text(new_text, encoding="utf-8")
|
||||
fixed += 1
|
||||
if verbose:
|
||||
print(f"[repair] cleared path_error for {zotero_key}")
|
||||
else:
|
||||
if verbose:
|
||||
print(f"[repair] {zotero_key} path still unresolved")
|
||||
else:
|
||||
if verbose:
|
||||
print(
|
||||
f"[repair] {zotero_key} has empty pdf_path (not_found)"
|
||||
)
|
||||
else:
|
||||
if verbose:
|
||||
print(f"[repair] {zotero_key} has no pdf_path field")
|
||||
|
||||
return fixed
|
||||
|
||||
|
||||
def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = False, fix_paths: bool = False) -> dict:
|
||||
"""Scan all domains for three-way state divergence and optionally repair.
|
||||
|
||||
Compares three sources of ocr_status:
|
||||
1. library_record.md frontmatter ocr_status
|
||||
2. formal_note.md frontmatter ocr_status
|
||||
3. meta.json ocr_status (post-validate_ocr_meta())
|
||||
|
||||
Returns:
|
||||
dict with scanned, divergent, fixed, errors counts
|
||||
"""
|
||||
result = {"scanned": 0, "divergent": [], "fixed": 0, "errors": []}
|
||||
config = load_domain_config(paths)
|
||||
domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']}
|
||||
record_paths = list(paths['library_records'].rglob('*.md'))
|
||||
for record_path in record_paths:
|
||||
try:
|
||||
record_text = record_path.read_text(encoding='utf-8')
|
||||
except Exception as e:
|
||||
result['errors'].append({"file": str(record_path), "error": str(e)})
|
||||
continue
|
||||
key_match = re.search('^zotero_key:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE)
|
||||
if not key_match:
|
||||
continue
|
||||
zotero_key = key_match.group(1).strip()
|
||||
domain = record_path.parent.name
|
||||
record_dir = record_path.parent
|
||||
result['scanned'] += 1
|
||||
lib_ocr_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE)
|
||||
lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else 'pending'
|
||||
note_path = _resolve_formal_note_path(vault, zotero_key, domain)
|
||||
note_ocr_status = None
|
||||
if note_path and note_path.exists():
|
||||
try:
|
||||
note_text = note_path.read_text(encoding='utf-8')
|
||||
note_status_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', note_text, re.MULTILINE)
|
||||
note_ocr_status = note_status_match.group(1).strip() if note_status_match else None
|
||||
except Exception:
|
||||
pass
|
||||
meta_path = paths['ocr'] / zotero_key / 'meta.json'
|
||||
meta_ocr_status = None
|
||||
meta_validated_status = None
|
||||
validated_status = None
|
||||
validated_error = ""
|
||||
if meta_path.exists():
|
||||
try:
|
||||
meta = read_json(meta_path)
|
||||
validated_status, validated_error = validate_ocr_meta(paths, meta)
|
||||
meta_validated_status = validated_status
|
||||
if validated_error and verbose:
|
||||
print(f"[repair] {zotero_key} meta validation error: {validated_error}")
|
||||
raw_status = str(meta.get('ocr_status', '') or '').strip().lower()
|
||||
meta_ocr_status = raw_status if raw_status else None
|
||||
if meta_validated_status == 'done_incomplete':
|
||||
meta_ocr_status = 'done_incomplete'
|
||||
except Exception as e:
|
||||
result['errors'].append({"file": str(meta_path), "error": str(e)})
|
||||
meta_ocr_status = None
|
||||
is_divergent = False
|
||||
div_reason = ""
|
||||
if meta_validated_status == 'done_incomplete':
|
||||
is_divergent = True
|
||||
div_reason = f"meta validation: done_incomplete ({validated_error})"
|
||||
elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing', None):
|
||||
is_divergent = True
|
||||
div_reason = f"library_record done but meta {meta_ocr_status or 'missing'}"
|
||||
elif note_ocr_status == 'done' and (meta_ocr_status is None or meta_validated_status == 'done_incomplete'):
|
||||
is_divergent = True
|
||||
div_reason = "formal_note done but meta.json missing/invalid"
|
||||
elif lib_ocr_status != 'pending' and meta_ocr_status is not None and meta_validated_status is not None and lib_ocr_status != meta_validated_status:
|
||||
is_divergent = True
|
||||
div_reason = f"library_record={lib_ocr_status} vs meta post-validation={meta_validated_status}"
|
||||
if is_divergent:
|
||||
item = {
|
||||
"zotero_key": zotero_key,
|
||||
"domain": domain,
|
||||
"library_record_ocr_status": lib_ocr_status,
|
||||
"formal_note_ocr_status": note_ocr_status,
|
||||
"meta_ocr_status": meta_validated_status or meta_ocr_status,
|
||||
"reason": div_reason,
|
||||
}
|
||||
result['divergent'].append(item)
|
||||
if verbose:
|
||||
print(f"[repair] divergent: {zotero_key} | {div_reason}")
|
||||
if fix:
|
||||
fixed_library_record = False
|
||||
fixed_formal_note = False
|
||||
fixed_meta = False
|
||||
new_status = 'pending'
|
||||
if meta_ocr_status is None or meta_validated_status == 'done_incomplete':
|
||||
new_status = 'pending'
|
||||
new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status)
|
||||
if new_record_text != record_text:
|
||||
record_path.write_text(new_record_text, encoding='utf-8')
|
||||
fixed_library_record = True
|
||||
if note_path and note_path.exists():
|
||||
try:
|
||||
note_text = note_path.read_text(encoding='utf-8')
|
||||
new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status)
|
||||
if new_note_text != note_text:
|
||||
note_path.write_text(new_note_text, encoding='utf-8')
|
||||
fixed_formal_note = True
|
||||
except Exception:
|
||||
pass
|
||||
if meta_validated_status is not None and meta_validated_status != 'done':
|
||||
if meta_path.exists():
|
||||
try:
|
||||
meta = read_json(meta_path)
|
||||
meta['ocr_status'] = 'pending'
|
||||
write_json(meta_path, meta)
|
||||
fixed_meta = True
|
||||
except Exception:
|
||||
pass
|
||||
record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE)
|
||||
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true'
|
||||
if not is_do_ocr:
|
||||
final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true')
|
||||
if final_record_text != new_record_text:
|
||||
record_path.write_text(final_record_text, encoding='utf-8')
|
||||
fixed_library_record = True
|
||||
elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing'):
|
||||
new_status = 'pending'
|
||||
new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status)
|
||||
if new_record_text != record_text:
|
||||
record_path.write_text(new_record_text, encoding='utf-8')
|
||||
fixed_library_record = True
|
||||
if note_path and note_path.exists():
|
||||
try:
|
||||
note_text = note_path.read_text(encoding='utf-8')
|
||||
new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status)
|
||||
if new_note_text != note_text:
|
||||
note_path.write_text(new_note_text, encoding='utf-8')
|
||||
fixed_formal_note = True
|
||||
except Exception:
|
||||
pass
|
||||
if meta_path.exists():
|
||||
try:
|
||||
meta = read_json(meta_path)
|
||||
meta['ocr_status'] = 'pending'
|
||||
write_json(meta_path, meta)
|
||||
fixed_meta = True
|
||||
except Exception:
|
||||
pass
|
||||
record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE)
|
||||
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true'
|
||||
if not is_do_ocr:
|
||||
final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true')
|
||||
if final_record_text != new_record_text:
|
||||
record_path.write_text(final_record_text, encoding='utf-8')
|
||||
fixed_library_record = True
|
||||
fixed_count = sum([fixed_library_record, fixed_formal_note, fixed_meta])
|
||||
result['fixed'] += fixed_count
|
||||
if verbose and fixed_count > 0:
|
||||
print(f"[repair] fixed {fixed_count} files for {zotero_key}")
|
||||
# Path error detection and repair
|
||||
path_errors = _detect_path_errors(paths, verbose)
|
||||
if path_errors["total"] > 0:
|
||||
error_summary = ", ".join(
|
||||
f"{count} {err}" for err, count in sorted(path_errors["by_type"].items())
|
||||
)
|
||||
print(
|
||||
f"[repair] Found {path_errors['total']} items with path errors: {error_summary}"
|
||||
)
|
||||
if fix_paths:
|
||||
fixed_count = repair_pdf_paths(
|
||||
vault, paths, path_errors["records"], verbose
|
||||
)
|
||||
print(f"[repair] Fixed {fixed_count} PDF paths")
|
||||
else:
|
||||
print("[repair] Tip: run with --fix-paths to attempt auto-resolution")
|
||||
elif verbose:
|
||||
print("[repair] No path errors found")
|
||||
|
||||
result["path_errors"] = path_errors
|
||||
return result
|
||||
|
||||
631
paperforge/worker/status.py
Normal file
631
paperforge/worker/status.py
Normal file
|
|
@ -0,0 +1,631 @@
|
|||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from json import JSONDecodeError
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
import requests
|
||||
import fitz
|
||||
from PIL import Image
|
||||
|
||||
from paperforge.worker.base_views import ensure_base_views
|
||||
|
||||
STANDARD_VIEW_NAMES = frozenset([
|
||||
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
|
||||
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
|
||||
])
|
||||
|
||||
def load_simple_env(env_path: Path) -> None:
|
||||
if not env_path.exists():
|
||||
return
|
||||
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
key = key.strip()
|
||||
if not key or key in os.environ:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
def read_json(path: Path):
|
||||
return json.loads(path.read_text(encoding='utf-8'))
|
||||
_JOURNAL_DB: dict[str, dict] | None = None
|
||||
|
||||
def load_journal_db(vault: Path) -> dict[str, dict]:
|
||||
"""Load zoterostyle.json journal database."""
|
||||
global _JOURNAL_DB
|
||||
if _JOURNAL_DB is not None:
|
||||
return _JOURNAL_DB
|
||||
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
|
||||
if zoterostyle_path.exists():
|
||||
try:
|
||||
_JOURNAL_DB = read_json(zoterostyle_path)
|
||||
except (JSONDecodeError, Exception):
|
||||
_JOURNAL_DB = {}
|
||||
else:
|
||||
_JOURNAL_DB = {}
|
||||
return _JOURNAL_DB
|
||||
|
||||
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
|
||||
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
|
||||
if not journal_name:
|
||||
return ''
|
||||
journal_db = load_journal_db(vault)
|
||||
if journal_name in journal_db:
|
||||
rank_data = journal_db[journal_name].get('rank', {})
|
||||
if isinstance(rank_data, dict):
|
||||
sciif = rank_data.get('sciif', '')
|
||||
if sciif:
|
||||
return str(sciif)
|
||||
if extra:
|
||||
if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra)
|
||||
if if_match:
|
||||
return if_match.group(1)
|
||||
return ''
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.read_text(encoding='utf-8').splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
def write_jsonl(path: Path, rows) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
|
||||
if text:
|
||||
text += '\n'
|
||||
path.write_text(text, encoding='utf-8')
|
||||
|
||||
def yaml_quote(value: str) -> str:
|
||||
if isinstance(value, bool):
|
||||
return 'true' if value else 'false'
|
||||
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
def yaml_block(value: str) -> list[str]:
|
||||
value = (value or '').strip()
|
||||
if not value:
|
||||
return ['abstract: |-', ' ']
|
||||
lines = ['abstract: |-']
|
||||
for line in value.splitlines():
|
||||
lines.append(f' {line}')
|
||||
return lines
|
||||
|
||||
def yaml_list(key: str, values) -> list[str]:
|
||||
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
|
||||
if not cleaned:
|
||||
return [f'{key}: []']
|
||||
lines = [f'{key}:']
|
||||
for value in cleaned:
|
||||
lines.append(f' - {yaml_quote(value)}')
|
||||
return lines
|
||||
|
||||
def slugify_filename(text: str) -> str:
|
||||
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
|
||||
return cleaned[:120] or 'untitled'
|
||||
|
||||
def _extract_year(value: str) -> str:
|
||||
match = re.search('(19|20)\\d{2}', value or '')
|
||||
return match.group(0) if match else ''
|
||||
|
||||
|
||||
def load_vault_config(vault: Path) -> dict:
|
||||
"""Read vault configuration — delegates to shared resolver.
|
||||
|
||||
Preserves the public name for legacy callers. Configuration precedence:
|
||||
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
|
||||
"""
|
||||
from paperforge.config import load_vault_config as _shared_load_vault_config
|
||||
return _shared_load_vault_config(vault)
|
||||
|
||||
|
||||
def pipeline_paths(vault: Path) -> dict[str, Path]:
|
||||
"""Build complete PaperForge path inventory — delegates to shared resolver.
|
||||
|
||||
Returns paths from paperforge.config.paperforge_paths() plus
|
||||
worker-only keys. Preserves all legacy keys for existing callers.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths as _shared_paperforge_paths
|
||||
|
||||
shared = _shared_paperforge_paths(vault)
|
||||
|
||||
cfg = load_vault_config(vault)
|
||||
system_dir = cfg["system_dir"]
|
||||
resources_dir = cfg["resources_dir"]
|
||||
control_dir = cfg["control_dir"]
|
||||
|
||||
root = shared["paperforge"]
|
||||
control_root = shared["control"]
|
||||
|
||||
return {
|
||||
**shared,
|
||||
# Worker-only keys (added on top of shared resolver output)
|
||||
"pipeline": root,
|
||||
"candidates": root / "candidates" / "candidates.json",
|
||||
"candidate_inbox": root / "candidates" / "inbox",
|
||||
"candidate_archive": root / "candidates" / "archive",
|
||||
"search_tasks": root / "search" / "tasks",
|
||||
"search_archive": root / "search" / "archive",
|
||||
"search_results": root / "search" / "results",
|
||||
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
|
||||
"records": control_root / "candidate-records",
|
||||
"review": root / "candidates" / "review-latest.md",
|
||||
"config": root / "config" / "domain-collections.json",
|
||||
"queue": root / "writeback" / "writeback-queue.jsonl",
|
||||
"log": root / "writeback" / "writeback-log.jsonl",
|
||||
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
|
||||
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
|
||||
"index": root / "indexes" / "formal-library.json",
|
||||
"ocr_queue": root / "ocr" / "ocr-queue.json",
|
||||
}
|
||||
|
||||
def load_domain_config(paths: dict[str, Path]) -> dict:
|
||||
"""Load or create the Lite domain mapping from export JSON files."""
|
||||
config_path = paths['config']
|
||||
if config_path.exists():
|
||||
config = read_json(config_path)
|
||||
else:
|
||||
config = {"domains": []}
|
||||
domains = config.setdefault("domains", [])
|
||||
known_exports = {str(entry.get("export_file", "")) for entry in domains}
|
||||
changed = not config_path.exists()
|
||||
for export_path in sorted(paths['exports'].glob('*.json')):
|
||||
if export_path.name in known_exports:
|
||||
continue
|
||||
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
|
||||
known_exports.add(export_path.name)
|
||||
changed = True
|
||||
if changed:
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json(config_path, config)
|
||||
return config
|
||||
|
||||
def _detect_zotero_data_dir() -> str | None:
|
||||
"""Try to detect the user's Zotero data directory."""
|
||||
if os.name == "nt":
|
||||
appdata = os.environ.get("APPDATA", "")
|
||||
if appdata:
|
||||
candidate = Path(appdata) / "Zotero"
|
||||
if candidate.exists() and (candidate / "storage").exists():
|
||||
return str(candidate)
|
||||
home = Path.home()
|
||||
candidates = []
|
||||
if os.name == "nt":
|
||||
candidates = [
|
||||
home / "Zotero",
|
||||
home / "AppData" / "Roaming" / "Zotero",
|
||||
]
|
||||
else:
|
||||
candidates = [
|
||||
home / "Zotero",
|
||||
home / ".zotero" / "zotero",
|
||||
home / "Library" / "Application Support" / "Zotero",
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate.exists() and (candidate / "storage").exists():
|
||||
return str(candidate)
|
||||
return None
|
||||
|
||||
|
||||
def check_zotero_location(vault: Path, cfg: dict, add_check) -> None:
|
||||
"""Detect if Zotero data directory is inside vault or linked via junction."""
|
||||
system_dir = vault / cfg["system_dir"]
|
||||
zotero_link = system_dir / "Zotero"
|
||||
if zotero_link.exists():
|
||||
if zotero_link.is_symlink() or _is_junction(zotero_link):
|
||||
try:
|
||||
target = os.path.realpath(zotero_link)
|
||||
if Path(target).exists():
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"pass",
|
||||
f"Zotero junction valid -> {target}",
|
||||
)
|
||||
else:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"warn",
|
||||
f"Zotero junction target missing: {target}",
|
||||
f"Recreate junction: mklink /J \"{zotero_link}\" \"<Zotero数据目录>\"",
|
||||
)
|
||||
except Exception:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"warn",
|
||||
"Zotero junction exists but target could not be resolved",
|
||||
)
|
||||
else:
|
||||
if (zotero_link / "storage").exists():
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"pass",
|
||||
"Zotero inside vault -- direct paths available",
|
||||
)
|
||||
else:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"warn",
|
||||
"Zotero directory exists but missing storage/ subdirectory",
|
||||
"Verify this is a valid Zotero data directory",
|
||||
)
|
||||
else:
|
||||
zotero_data_dir = _detect_zotero_data_dir()
|
||||
if zotero_data_dir:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"warn",
|
||||
"Zotero outside vault -- junction recommended",
|
||||
f'Run as Administrator: mklink /J "{zotero_link}" "{zotero_data_dir}"',
|
||||
)
|
||||
else:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"fail",
|
||||
"Zotero directory not found",
|
||||
f'Run as Administrator: mklink /J "{zotero_link}" "<Zotero数据目录>"',
|
||||
)
|
||||
|
||||
|
||||
def _summarize_errors(errors: list[str]) -> str:
|
||||
"""Summarize error types into a human-readable string."""
|
||||
counts: dict[str, int] = {}
|
||||
for e in errors:
|
||||
counts[e] = counts.get(e, 0) + 1
|
||||
return ", ".join(f"{count} {err}" for err, count in sorted(counts.items()))
|
||||
|
||||
|
||||
def check_pdf_paths(vault: Path, paths: dict, add_check) -> None:
|
||||
"""Sample up to 5 library records and validate their pdf_path wikilinks."""
|
||||
record_paths = list(paths["library_records"].rglob("*.md"))
|
||||
if not record_paths:
|
||||
add_check("Path Resolution", "pass", "No library records to check")
|
||||
return
|
||||
import random
|
||||
|
||||
sample = (
|
||||
record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5)
|
||||
)
|
||||
valid = 0
|
||||
errors: list[str] = []
|
||||
for record_path in sample:
|
||||
try:
|
||||
text = record_path.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
continue
|
||||
pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE)
|
||||
if not pdf_match:
|
||||
continue
|
||||
pdf_path = pdf_match.group(1).strip()
|
||||
if not pdf_path:
|
||||
continue
|
||||
raw_path = pdf_path.strip("[]")
|
||||
candidate = vault / raw_path.replace("/", os.sep)
|
||||
if candidate.exists():
|
||||
valid += 1
|
||||
else:
|
||||
err_match = re.search(
|
||||
r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE
|
||||
)
|
||||
error_type = err_match.group(1) if err_match else "not_found"
|
||||
errors.append(error_type)
|
||||
total = len(sample)
|
||||
if valid == total:
|
||||
add_check("Path Resolution", "pass", f"{valid}/{total} PDF paths valid")
|
||||
else:
|
||||
error_summary = _summarize_errors(errors) if errors else "unknown"
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"warn",
|
||||
f"{valid}/{total} PDF paths valid, {total - valid} path errors: {error_summary}",
|
||||
)
|
||||
|
||||
|
||||
def check_wikilink_format(vault: Path, paths: dict, add_check) -> None:
|
||||
"""Verify all pdf_path values in library-records use [[...]] format."""
|
||||
record_paths = list(paths["library_records"].rglob("*.md"))
|
||||
bad_paths: list[str] = []
|
||||
for record_path in record_paths:
|
||||
try:
|
||||
text = record_path.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
continue
|
||||
pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE)
|
||||
if not pdf_match:
|
||||
continue
|
||||
pdf_path = pdf_match.group(1).strip()
|
||||
if not pdf_path:
|
||||
continue
|
||||
if not (pdf_path.startswith("[[") and pdf_path.endswith("]]")):
|
||||
bad_paths.append(str(record_path.relative_to(vault)))
|
||||
if bad_paths:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"warn",
|
||||
f"{len(bad_paths)} pdf_path values not in wikilink format",
|
||||
"Re-run `paperforge sync` to regenerate wikilinks",
|
||||
)
|
||||
else:
|
||||
add_check(
|
||||
"Path Resolution",
|
||||
"pass",
|
||||
"All pdf_path values use wikilink format",
|
||||
)
|
||||
|
||||
|
||||
def run_doctor(vault: Path) -> int:
|
||||
"""Validate PaperForge Lite setup and report by category.
|
||||
|
||||
Returns:
|
||||
0 if all checks pass, 1 otherwise.
|
||||
"""
|
||||
paths = pipeline_paths(vault)
|
||||
cfg = load_vault_config(vault)
|
||||
checks: list[tuple[str, str, str, str]] = []
|
||||
|
||||
def add_check(category: str, status: str, message: str, fix: str = "") -> None:
|
||||
checks.append((category, status, message, fix))
|
||||
|
||||
sys_version = sys.version_info
|
||||
if sys_version.major >= 3 and sys_version.minor >= 10:
|
||||
add_check("Python 环境", "pass", f"Python {sys_version.major}.{sys_version.minor}.{sys_version.micro}")
|
||||
else:
|
||||
add_check("Python 环境", "fail", f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)", "升级 Python 到 3.10 或更高版本")
|
||||
|
||||
required_modules = ["requests", "pymupdf", "PIL", "yaml"]
|
||||
missing_modules = []
|
||||
for mod in required_modules:
|
||||
try:
|
||||
__import__(mod)
|
||||
except ImportError:
|
||||
missing_modules.append(mod)
|
||||
if missing_modules:
|
||||
add_check("Python 环境", "fail", f"缺少模块: {', '.join(missing_modules)}", f"运行: pip install {' '.join(missing_modules)}")
|
||||
elif sys_version.major >= 3 and sys_version.minor >= 10:
|
||||
pass
|
||||
|
||||
if (vault / "paperforge.json").exists():
|
||||
add_check("Vault 结构", "pass", "paperforge.json 存在")
|
||||
else:
|
||||
add_check("Vault 结构", "fail", "paperforge.json 不存在", "在 vault 根目录创建 paperforge.json")
|
||||
|
||||
system_dir = vault / cfg["system_dir"]
|
||||
if system_dir.exists():
|
||||
add_check("Vault 结构", "pass", f"system_dir 存在: {cfg['system_dir']}")
|
||||
else:
|
||||
add_check("Vault 结构", "fail", f"system_dir 不存在: {cfg['system_dir']}", f"运行: mkdir {cfg['system_dir']}")
|
||||
|
||||
resources_dir = vault / cfg["resources_dir"]
|
||||
if resources_dir.exists():
|
||||
add_check("Vault 结构", "pass", f"resources_dir 存在: {cfg['resources_dir']}")
|
||||
else:
|
||||
add_check("Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}")
|
||||
|
||||
control_dir = resources_dir / cfg.get("control_dir", "LiteratureControl")
|
||||
if control_dir.exists():
|
||||
add_check("Vault 结构", "pass", f"control_dir 存在: {cfg.get('control_dir', 'LiteratureControl')}")
|
||||
else:
|
||||
add_check("Vault 结构", "fail", f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}", f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}")
|
||||
|
||||
zotero_link = system_dir / "Zotero"
|
||||
if zotero_link.exists():
|
||||
if zotero_link.is_symlink() or _is_junction(zotero_link):
|
||||
add_check("Zotero 链接", "pass", "Zotero 目录是 junction/symlink")
|
||||
else:
|
||||
add_check("Zotero 链接", "warn", "Zotero 目录存在但不是 junction,建议使用 junction")
|
||||
else:
|
||||
add_check("Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} <Zotero数据目录>")
|
||||
|
||||
exports_dir = system_dir / "PaperForge" / "exports"
|
||||
if exports_dir.exists():
|
||||
add_check("BBT 导出", "pass", "exports 目录存在")
|
||||
else:
|
||||
add_check("BBT 导出", "fail", "exports 目录不存在", "在 Better BibTeX 设置中配置导出路径")
|
||||
|
||||
# Check for any valid JSON export (per-domain or library.json)
|
||||
json_files = sorted(exports_dir.glob("*.json")) if exports_dir.exists() else []
|
||||
valid_exports = []
|
||||
for jf in json_files:
|
||||
try:
|
||||
data = json.loads(jf.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list) and len(data) > 0:
|
||||
has_key = any("key" in item or "citation-key" in item for item in data[:5])
|
||||
if has_key:
|
||||
valid_exports.append((jf.name, len(data)))
|
||||
elif isinstance(data, dict) and isinstance(data.get("items"), list) and len(data["items"]) > 0:
|
||||
has_key = any("key" in item or "citation-key" in item for item in data["items"][:5])
|
||||
if has_key:
|
||||
valid_exports.append((jf.name, len(data["items"])))
|
||||
except (JSONDecodeError, Exception):
|
||||
pass
|
||||
|
||||
if valid_exports:
|
||||
for name, count in valid_exports:
|
||||
add_check("BBT 导出", "pass", f"{name} 正常 ({count} 条)")
|
||||
elif json_files:
|
||||
add_check("BBT 导出", "warn", "导出文件存在但无有效 citation key")
|
||||
else:
|
||||
add_check("BBT 导出", "fail", "未找到 JSON 导出文件", "在 Zotero Better BibTeX 设置中配置导出路径")
|
||||
|
||||
env_api_key = os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN")
|
||||
if env_api_key:
|
||||
add_check("OCR 配置", "pass", "API Token 已配置")
|
||||
else:
|
||||
add_check("OCR 配置", "fail", "缺少 PADDLEOCR_API_TOKEN", "在 .env 文件中设置 PADDLEOCR_API_TOKEN")
|
||||
|
||||
if (paths["pipeline"] / "literature_pipeline.py").exists():
|
||||
add_check("Worker 脚本", "pass", "literature_pipeline.py 存在")
|
||||
try:
|
||||
from paperforge.worker.sync import (
|
||||
run_selection_sync, run_index_refresh,
|
||||
)
|
||||
from paperforge.worker.deep_reading import run_deep_reading
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
from paperforge.worker.base_views import ensure_base_views
|
||||
run_status # self-reference
|
||||
add_check("Worker 脚本", "pass", "所有 worker 函数可导入")
|
||||
except ImportError as e:
|
||||
add_check("Worker 脚本", "fail", f"worker 函数导入失败: {e}", "检查 pipeline/worker/scripts/literature_pipeline.py")
|
||||
else:
|
||||
add_check("Worker 脚本", "fail", "literature_pipeline.py 不存在", "确认 PaperForge pipeline 已正确安装")
|
||||
|
||||
# Path Resolution checks
|
||||
check_zotero_location(vault, cfg, add_check)
|
||||
check_pdf_paths(vault, paths, add_check)
|
||||
check_wikilink_format(vault, paths, add_check)
|
||||
|
||||
ld_deep_script = paths.get("ld_deep_script")
|
||||
skill_dir = None
|
||||
if ld_deep_script:
|
||||
skill_dir = ld_deep_script.parent.parent
|
||||
if skill_dir and skill_dir.exists():
|
||||
# Try actual importability check
|
||||
ld_deep_import_ok = False
|
||||
import_error = ""
|
||||
if ld_deep_script and ld_deep_script.exists():
|
||||
try:
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("ld_deep", ld_deep_script)
|
||||
if spec and spec.loader:
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
ld_deep_import_ok = True
|
||||
except Exception as e:
|
||||
import_error = str(e)
|
||||
if ld_deep_import_ok:
|
||||
add_check("Agent 脚本", "pass", "paperforge and ld_deep importable")
|
||||
else:
|
||||
add_check("Agent 脚本", "warn", f"literature-qa skill 目录存在但 import 失败: {import_error}", "确认 agent_config_dir 配置正确并已运行 pip install -e .")
|
||||
else:
|
||||
add_check("Agent 脚本", "warn", "literature-qa skill 目录未找到", "确认 agent_config_dir 配置正确")
|
||||
|
||||
print("PaperForge Lite Doctor")
|
||||
print("=" * 40)
|
||||
current_category = ""
|
||||
fix_map: dict[str, list[str]] = {}
|
||||
for category, status, message, fix in checks:
|
||||
if category != current_category:
|
||||
if current_category:
|
||||
print()
|
||||
current_category = category
|
||||
status_tag = {"pass": "[PASS]", "fail": "[FAIL]", "warn": "[WARN]"}[status]
|
||||
print(f"{status_tag} {category} — {message}")
|
||||
if status == "fail" and fix:
|
||||
fix_map.setdefault(category, [])
|
||||
fix_map[category].append(fix)
|
||||
|
||||
if fix_map:
|
||||
print("\n修复步骤:")
|
||||
for cat, fixes in fix_map.items():
|
||||
for f in fixes:
|
||||
print(f" - {cat}: {f}")
|
||||
print()
|
||||
return 1
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
def _is_junction(path: Path) -> bool:
|
||||
"""Check if a path is a Windows junction point."""
|
||||
try:
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
|
||||
GetFileAttributesW = ctypes.windll.kernel32.GetFileAttributesW
|
||||
GetFileAttributesW.argtypes = [wintypes.LPCWSTR]
|
||||
GetFileAttributesW.restype = wintypes.DWORD
|
||||
attrs = GetFileAttributesW(str(path))
|
||||
if attrs == INVALID_FILE_ATTRIBUTES:
|
||||
return False
|
||||
return bool(attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def run_status(vault: Path) -> int:
|
||||
"""Print a compact Lite install/runtime status."""
|
||||
paths = pipeline_paths(vault)
|
||||
cfg = load_vault_config(vault)
|
||||
config = load_domain_config(paths)
|
||||
ensure_base_views(vault, paths, config)
|
||||
export_files = sorted(paths['exports'].glob('*.json'))
|
||||
record_count = sum(1 for _ in paths['library_records'].rglob('*.md')) if paths['library_records'].exists() else 0
|
||||
note_count = sum(1 for _ in paths['literature'].rglob('*.md')) if paths['literature'].exists() else 0
|
||||
base_count = sum(1 for _ in paths['bases'].glob('*.base')) if paths['bases'].exists() else 0
|
||||
ocr_done = 0
|
||||
ocr_total = 0
|
||||
if paths['ocr'].exists():
|
||||
for meta_path in paths['ocr'].glob('*/meta.json'):
|
||||
ocr_total += 1
|
||||
try:
|
||||
meta = read_json(meta_path)
|
||||
except Exception:
|
||||
continue
|
||||
if str(meta.get('ocr_status', '')).strip().lower() == 'done':
|
||||
ocr_done += 1
|
||||
env_paths = [vault / '.env', paths['pipeline'] / '.env']
|
||||
env_found = [str(path.relative_to(vault)).replace('\\', '/') for path in env_paths if path.exists()]
|
||||
|
||||
# Count path errors
|
||||
path_error_count = 0
|
||||
if paths['library_records'].exists():
|
||||
for record_path in paths['library_records'].rglob('*.md'):
|
||||
try:
|
||||
text = record_path.read_text(encoding='utf-8')
|
||||
if re.search(r'^path_error:\s*"(.+?)"\s*$', text, re.MULTILINE):
|
||||
path_error_count += 1
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
print('PaperForge Lite status')
|
||||
print(f"- vault: {vault}")
|
||||
print(f"- system_dir: {cfg['system_dir']}")
|
||||
print(f"- resources_dir: {cfg['resources_dir']}")
|
||||
print(f"- literature_dir: {cfg['literature_dir']}")
|
||||
print(f"- control_dir: {cfg['control_dir']}")
|
||||
print(f"- exports: {len(export_files)} JSON file(s)")
|
||||
print(f"- domains: {len(config.get('domains', []))}")
|
||||
print(f"- library_records: {record_count}")
|
||||
print(f"- formal_notes: {note_count}")
|
||||
print(f"- bases: {base_count}")
|
||||
print(f"- ocr: {ocr_done}/{ocr_total} done")
|
||||
print(f"- path_errors: {path_error_count}")
|
||||
if path_error_count > 0:
|
||||
print(" Tip: Run `paperforge repair --fix-paths` to attempt resolution")
|
||||
print(f"- env: {', '.join(env_found) if env_found else 'not configured'}")
|
||||
return 0
|
||||
|
||||
# =============================================================================
|
||||
# Update 功能
|
||||
# =============================================================================
|
||||
|
||||
GITHUB_REPO = "LLLin000/PaperForge"
|
||||
GITHUB_ZIP = f"https://github.com/{GITHUB_REPO}/archive/refs/heads/master.zip"
|
||||
|
||||
UPDATEABLE_PATHS = ["skills", "pipeline", "command", "scripts"]
|
||||
|
||||
|
||||
1444
paperforge/worker/sync.py
Normal file
1444
paperforge/worker/sync.py
Normal file
File diff suppressed because it is too large
Load diff
398
paperforge/worker/update.py
Normal file
398
paperforge/worker/update.py
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
from json import JSONDecodeError
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree as ET
|
||||
import requests
|
||||
import fitz
|
||||
from PIL import Image
|
||||
|
||||
STANDARD_VIEW_NAMES = frozenset([
|
||||
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
|
||||
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
|
||||
])
|
||||
|
||||
def load_simple_env(env_path: Path) -> None:
|
||||
if not env_path.exists():
|
||||
return
|
||||
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, value = line.split('=', 1)
|
||||
key = key.strip()
|
||||
if not key or key in os.environ:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
|
||||
value = value[1:-1]
|
||||
os.environ[key] = value
|
||||
|
||||
def read_json(path: Path):
|
||||
return json.loads(path.read_text(encoding='utf-8'))
|
||||
_JOURNAL_DB: dict[str, dict] | None = None
|
||||
|
||||
def load_journal_db(vault: Path) -> dict[str, dict]:
|
||||
"""Load zoterostyle.json journal database."""
|
||||
global _JOURNAL_DB
|
||||
if _JOURNAL_DB is not None:
|
||||
return _JOURNAL_DB
|
||||
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
|
||||
if zoterostyle_path.exists():
|
||||
try:
|
||||
_JOURNAL_DB = read_json(zoterostyle_path)
|
||||
except (JSONDecodeError, Exception):
|
||||
_JOURNAL_DB = {}
|
||||
else:
|
||||
_JOURNAL_DB = {}
|
||||
return _JOURNAL_DB
|
||||
|
||||
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
|
||||
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
|
||||
if not journal_name:
|
||||
return ''
|
||||
journal_db = load_journal_db(vault)
|
||||
if journal_name in journal_db:
|
||||
rank_data = journal_db[journal_name].get('rank', {})
|
||||
if isinstance(rank_data, dict):
|
||||
sciif = rank_data.get('sciif', '')
|
||||
if sciif:
|
||||
return str(sciif)
|
||||
if extra:
|
||||
if_match = re.search('影响因子[::]\\s*([0-9.]+)', extra)
|
||||
if if_match:
|
||||
return if_match.group(1)
|
||||
return ''
|
||||
|
||||
def write_json(path: Path, data) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
def read_jsonl(path: Path):
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.read_text(encoding='utf-8').splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
def write_jsonl(path: Path, rows) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
|
||||
if text:
|
||||
text += '\n'
|
||||
path.write_text(text, encoding='utf-8')
|
||||
|
||||
def yaml_quote(value: str) -> str:
|
||||
if isinstance(value, bool):
|
||||
return 'true' if value else 'false'
|
||||
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
def yaml_block(value: str) -> list[str]:
|
||||
value = (value or '').strip()
|
||||
if not value:
|
||||
return ['abstract: |-', ' ']
|
||||
lines = ['abstract: |-']
|
||||
for line in value.splitlines():
|
||||
lines.append(f' {line}')
|
||||
return lines
|
||||
|
||||
def yaml_list(key: str, values) -> list[str]:
|
||||
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
|
||||
if not cleaned:
|
||||
return [f'{key}: []']
|
||||
lines = [f'{key}:']
|
||||
for value in cleaned:
|
||||
lines.append(f' - {yaml_quote(value)}')
|
||||
return lines
|
||||
|
||||
def slugify_filename(text: str) -> str:
|
||||
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
|
||||
return cleaned[:120] or 'untitled'
|
||||
|
||||
def _extract_year(value: str) -> str:
|
||||
match = re.search('(19|20)\\d{2}', value or '')
|
||||
return match.group(0) if match else ''
|
||||
|
||||
|
||||
def load_vault_config(vault: Path) -> dict:
|
||||
"""Read vault configuration — delegates to shared resolver.
|
||||
|
||||
Preserves the public name for legacy callers. Configuration precedence:
|
||||
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
|
||||
"""
|
||||
from paperforge.config import load_vault_config as _shared_load_vault_config
|
||||
return _shared_load_vault_config(vault)
|
||||
|
||||
|
||||
def pipeline_paths(vault: Path) -> dict[str, Path]:
|
||||
"""Build complete PaperForge path inventory — delegates to shared resolver.
|
||||
|
||||
Returns paths from paperforge.config.paperforge_paths() plus
|
||||
worker-only keys. Preserves all legacy keys for existing callers.
|
||||
"""
|
||||
from paperforge.config import paperforge_paths as _shared_paperforge_paths
|
||||
|
||||
shared = _shared_paperforge_paths(vault)
|
||||
|
||||
cfg = load_vault_config(vault)
|
||||
system_dir = cfg["system_dir"]
|
||||
resources_dir = cfg["resources_dir"]
|
||||
control_dir = cfg["control_dir"]
|
||||
|
||||
root = shared["paperforge"]
|
||||
control_root = shared["control"]
|
||||
|
||||
return {
|
||||
**shared,
|
||||
# Worker-only keys (added on top of shared resolver output)
|
||||
"pipeline": root,
|
||||
"candidates": root / "candidates" / "candidates.json",
|
||||
"candidate_inbox": root / "candidates" / "inbox",
|
||||
"candidate_archive": root / "candidates" / "archive",
|
||||
"search_tasks": root / "search" / "tasks",
|
||||
"search_archive": root / "search" / "archive",
|
||||
"search_results": root / "search" / "results",
|
||||
"harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer",
|
||||
"records": control_root / "candidate-records",
|
||||
"review": root / "candidates" / "review-latest.md",
|
||||
"config": root / "config" / "domain-collections.json",
|
||||
"queue": root / "writeback" / "writeback-queue.jsonl",
|
||||
"log": root / "writeback" / "writeback-log.jsonl",
|
||||
"bridge_config": root / "zotero-bridge" / "bridge-config.json",
|
||||
"bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json",
|
||||
"index": root / "indexes" / "formal-library.json",
|
||||
"ocr_queue": root / "ocr" / "ocr-queue.json",
|
||||
}
|
||||
|
||||
def load_domain_config(paths: dict[str, Path]) -> dict:
|
||||
"""Load or create the Lite domain mapping from export JSON files."""
|
||||
config_path = paths['config']
|
||||
if config_path.exists():
|
||||
config = read_json(config_path)
|
||||
else:
|
||||
config = {"domains": []}
|
||||
domains = config.setdefault("domains", [])
|
||||
known_exports = {str(entry.get("export_file", "")) for entry in domains}
|
||||
changed = not config_path.exists()
|
||||
for export_path in sorted(paths['exports'].glob('*.json')):
|
||||
if export_path.name in known_exports:
|
||||
continue
|
||||
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
|
||||
known_exports.add(export_path.name)
|
||||
changed = True
|
||||
if changed:
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json(config_path, config)
|
||||
return config
|
||||
|
||||
def protected_paths(vault: Path) -> set[str]:
|
||||
cfg = load_vault_config(vault)
|
||||
pf = f"{cfg['system_dir']}/PaperForge"
|
||||
return {
|
||||
cfg["resources_dir"],
|
||||
cfg["base_dir"],
|
||||
f"{pf}/ocr",
|
||||
f"{pf}/exports",
|
||||
f"{pf}/indexes",
|
||||
f"{pf}/candidates",
|
||||
".env",
|
||||
"AGENTS.md",
|
||||
}
|
||||
|
||||
|
||||
def _color(text: str, c: str = "") -> str:
|
||||
colors = {"r": "\033[91m", "g": "\033[92m", "y": "\033[93m", "b": "\033[94m", "c": "\033[96m", "x": "\033[0m"}
|
||||
if sys.platform == "win32" and not os.environ.get("FORCE_COLOR"):
|
||||
return text
|
||||
return f"{colors.get(c, '')}{text}{colors['x']}"
|
||||
|
||||
|
||||
def _log(msg: str, c: str = "") -> None:
|
||||
print(_color(msg, c))
|
||||
|
||||
|
||||
def _remote_version() -> str | None:
|
||||
try:
|
||||
api = f"https://api.github.com/repos/{GITHUB_REPO}/contents/paperforge.json"
|
||||
req = urllib.request.Request(api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"})
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read())
|
||||
req2 = urllib.request.Request(data["download_url"], headers={"User-Agent": "PaperForge"})
|
||||
with urllib.request.urlopen(req2, timeout=10) as resp2:
|
||||
return json.loads(resp2.read()).get("version")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _scan_updates(vault: Path, source: Path) -> list[tuple[Path, Path, str]]:
|
||||
updates = []
|
||||
protected = protected_paths(vault)
|
||||
for name in UPDATEABLE_PATHS:
|
||||
src_dir = source / name
|
||||
if not src_dir.exists():
|
||||
continue
|
||||
for src in src_dir.rglob("*"):
|
||||
if not src.is_file():
|
||||
continue
|
||||
rel = src.relative_to(source)
|
||||
dst = vault / rel
|
||||
rel_str = str(rel).replace("\\", "/")
|
||||
if any(rel_str.startswith(p) for p in protected):
|
||||
continue
|
||||
if dst.exists():
|
||||
if hashlib.sha256(src.read_bytes()).hexdigest() != hashlib.sha256(dst.read_bytes()).hexdigest():
|
||||
updates.append((src, dst, "UPDATE"))
|
||||
else:
|
||||
updates.append((src, dst, "NEW"))
|
||||
return updates
|
||||
|
||||
|
||||
def _do_backup(vault: Path, updates: list) -> Path | None:
|
||||
backup_dir = vault / f".backup_{datetime.now():%Y%m%d_%H%M%S}"
|
||||
backup_dir.mkdir(exist_ok=True)
|
||||
count = 0
|
||||
for src, dst, action in updates:
|
||||
if action == "UPDATE" and dst.exists():
|
||||
bp = backup_dir / dst.relative_to(vault)
|
||||
bp.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(dst, bp)
|
||||
count += 1
|
||||
if count:
|
||||
_log(f"[INFO] 已备份 {count} 个文件到 {backup_dir.name}", "c")
|
||||
return backup_dir if count else None
|
||||
|
||||
|
||||
def _apply_updates(vault: Path, updates: list) -> bool:
|
||||
try:
|
||||
for src, dst, action in updates:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
return True
|
||||
except Exception as e:
|
||||
_log(f"[ERR] 更新失败: {e}", "r")
|
||||
return False
|
||||
|
||||
|
||||
def _rollback(vault: Path, backup_dir: Path) -> None:
|
||||
_log("[INFO] 正在回滚...", "b")
|
||||
for bp in backup_dir.rglob("*"):
|
||||
if bp.is_file():
|
||||
orig = vault / bp.relative_to(backup_dir)
|
||||
orig.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(bp, orig)
|
||||
_log("[OK] 回滚完成", "g")
|
||||
|
||||
|
||||
def update_via_git(vault: Path) -> bool:
|
||||
if not (vault / ".git").is_dir():
|
||||
_log("[ERR] 不是 git 仓库", "r")
|
||||
return False
|
||||
r = subprocess.run(["git", "status", "--short"], cwd=vault, capture_output=True, text=True, encoding="utf-8")
|
||||
if r.stdout.strip():
|
||||
_log("[WARN] 有未提交的更改,请先提交或储藏", "y")
|
||||
return False
|
||||
_log("[INFO] 执行 git pull...", "b")
|
||||
r = subprocess.run(["git", "pull", "origin", "master"], cwd=vault, capture_output=True, text=True, encoding="utf-8")
|
||||
if r.returncode != 0:
|
||||
_log(f"[ERR] git pull 失败: {r.stderr}", "r")
|
||||
return False
|
||||
_log("[OK] git pull 成功", "g")
|
||||
if r.stdout.strip():
|
||||
print(r.stdout)
|
||||
return True
|
||||
|
||||
|
||||
def update_via_zip(vault: Path) -> bool:
|
||||
_log("[INFO] 下载更新包...", "b")
|
||||
tmp = Path(tempfile.mkdtemp(prefix="pf_update_"))
|
||||
zip_path = tmp / "update.zip"
|
||||
try:
|
||||
req = urllib.request.Request(GITHUB_ZIP, headers={"User-Agent": "PaperForge"})
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
zip_path.write_bytes(resp.read())
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
zf.extractall(tmp / "extracted")
|
||||
dirs = [d for d in (tmp / "extracted").iterdir() if d.is_dir()]
|
||||
source = dirs[0] if dirs else None
|
||||
if not source:
|
||||
_log("[ERR] 解压失败", "r")
|
||||
return False
|
||||
updates = _scan_updates(vault, source)
|
||||
if not updates:
|
||||
_log("[OK] 所有文件已是最新", "g")
|
||||
return True
|
||||
_log(f"\n[INFO] 发现 {len(updates)} 个文件需要更新:", "b")
|
||||
for src, dst, action in updates:
|
||||
_log(f" [{action}] {dst.relative_to(vault)}", "g" if action == "NEW" else "y")
|
||||
backup = _do_backup(vault, updates)
|
||||
if _apply_updates(vault, updates):
|
||||
_log(f"\n[OK] 更新完成!共 {len(updates)} 个文件", "g")
|
||||
return True
|
||||
if backup:
|
||||
_rollback(vault, backup)
|
||||
return False
|
||||
except Exception as e:
|
||||
_log(f"[ERR] 下载失败: {e}", "r")
|
||||
return False
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def run_update(vault: Path) -> int:
|
||||
"""运行更新检查与安装"""
|
||||
local_cfg = vault / "paperforge.json"
|
||||
if not local_cfg.exists():
|
||||
_log("[ERR] 未找到 paperforge.json", "r")
|
||||
return 1
|
||||
local = json.loads(local_cfg.read_text(encoding="utf-8")).get("version", "unknown")
|
||||
remote = _remote_version()
|
||||
_log("=" * 50, "b")
|
||||
_log("PaperForge Lite 更新", "b")
|
||||
_log("=" * 50, "b")
|
||||
_log(f"本地版本: {local}", "c")
|
||||
_log(f"远程版本: {remote or 'unknown'}", "c")
|
||||
if not remote:
|
||||
_log("[ERR] 无法获取远程版本", "r")
|
||||
return 1
|
||||
try:
|
||||
needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple(int(x) for x in local.split(".") if x.isdigit())
|
||||
except ValueError:
|
||||
needs = remote != local
|
||||
if not needs:
|
||||
_log("[OK] 当前已是最新版本", "g")
|
||||
return 0
|
||||
_log(f"\n[INFO] 发现新版本: {local} -> {remote}", "y")
|
||||
_log("[WARN] 更新前建议备份 Vault", "y")
|
||||
ans = input(_color("确认更新? [y/N]: ", "y")).strip().lower()
|
||||
if ans not in ("y", "yes"):
|
||||
_log("[INFO] 已取消", "c")
|
||||
return 0
|
||||
if (vault / ".git").is_dir():
|
||||
success = update_via_git(vault)
|
||||
else:
|
||||
success = update_via_zip(vault)
|
||||
if success:
|
||||
_log("\n[OK] 更新完成!请重启 Obsidian", "g")
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main
|
||||
# =============================================================================
|
||||
|
||||
|
|
@ -929,7 +929,11 @@ class DeployStep(StepScreen):
|
|||
wizard_dir = Path(__file__).parent.resolve()
|
||||
# 如果 wizard 在 github-release/ 下,repo_root 就是 github-release/
|
||||
# 如果 wizard 在 scripts/ 下,repo_root 是父目录
|
||||
if (wizard_dir / "pipeline").exists():
|
||||
if (wizard_dir / "paperforge").exists():
|
||||
repo_root = wizard_dir
|
||||
elif (wizard_dir.parent / "paperforge").exists():
|
||||
repo_root = wizard_dir.parent
|
||||
elif (wizard_dir / "pipeline").exists():
|
||||
repo_root = wizard_dir
|
||||
elif (wizard_dir.parent / "pipeline").exists():
|
||||
repo_root = wizard_dir.parent
|
||||
|
|
@ -956,17 +960,31 @@ class DeployStep(StepScreen):
|
|||
# 4. 复制脚本(从安装包到 Vault)
|
||||
import shutil
|
||||
|
||||
# Copy pipeline worker to PaperForge/worker/scripts/
|
||||
worker_src = repo_root / "pipeline/worker/scripts/literature_pipeline.py"
|
||||
worker_dst = pf_path / "worker/scripts/literature_pipeline.py"
|
||||
# Copy worker modules to PaperForge/worker/scripts/
|
||||
worker_src = repo_root / "paperforge/worker/sync.py"
|
||||
worker_dst = pf_path / "worker/scripts/sync.py"
|
||||
if worker_src.exists():
|
||||
import shutil
|
||||
shutil.copy2(worker_src, worker_dst)
|
||||
# Also copy other worker modules
|
||||
for mod in ["ocr.py", "repair.py", "status.py", "deep_reading.py", "update.py", "base_views.py", "__init__.py"]:
|
||||
mod_src = repo_root / "paperforge/worker" / mod
|
||||
if mod_src.exists():
|
||||
shutil.copy2(mod_src, pf_path / "worker/scripts" / mod)
|
||||
else:
|
||||
self.set_status(f"错误:找不到 worker 脚本: {worker_src}", False)
|
||||
return False
|
||||
# Fallback to old pipeline location for backward compatibility
|
||||
worker_src = repo_root / "pipeline/worker/scripts/literature_pipeline.py"
|
||||
worker_dst = pf_path / "worker/scripts/literature_pipeline.py"
|
||||
if worker_src.exists():
|
||||
shutil.copy2(worker_src, worker_dst)
|
||||
else:
|
||||
self.set_status(f"错误:找不到 worker 脚本: {worker_src}", False)
|
||||
return False
|
||||
|
||||
# Copy ld_deep.py
|
||||
ld_src = repo_root / "skills/literature-qa/scripts/ld_deep.py"
|
||||
# Copy ld_deep.py (prefer paperforge/skills, fallback to skills/)
|
||||
ld_src = repo_root / "paperforge/skills/literature-qa/scripts/ld_deep.py"
|
||||
if not ld_src.exists():
|
||||
ld_src = repo_root / "skills/literature-qa/scripts/ld_deep.py"
|
||||
ld_dst = vault / skill_dir / "literature-qa/scripts/ld_deep.py"
|
||||
if ld_src.exists():
|
||||
shutil.copy2(ld_src, ld_dst)
|
||||
|
|
@ -974,8 +992,10 @@ class DeployStep(StepScreen):
|
|||
self.set_status(f"错误:找不到 ld_deep.py: {ld_src}", False)
|
||||
return False
|
||||
|
||||
# Copy subagent prompt
|
||||
prompt_src = repo_root / "skills/literature-qa/prompt_deep_subagent.md"
|
||||
# Copy subagent prompt (prefer paperforge/skills, fallback to skills/)
|
||||
prompt_src = repo_root / "paperforge/skills/literature-qa/prompt_deep_subagent.md"
|
||||
if not prompt_src.exists():
|
||||
prompt_src = repo_root / "skills/literature-qa/prompt_deep_subagent.md"
|
||||
prompt_dst = vault / skill_dir / "literature-qa/prompt_deep_subagent.md"
|
||||
if prompt_src.exists():
|
||||
shutil.copy2(prompt_src, prompt_dst)
|
||||
|
|
@ -983,8 +1003,10 @@ class DeployStep(StepScreen):
|
|||
self.set_status(f"错误:找不到 prompt_deep_subagent.md: {prompt_src}", False)
|
||||
return False
|
||||
|
||||
# Copy chart-reading guides
|
||||
chart_src = repo_root / "skills/literature-qa/chart-reading"
|
||||
# Copy chart-reading guides (prefer paperforge/skills, fallback to skills/)
|
||||
chart_src = repo_root / "paperforge/skills/literature-qa/chart-reading"
|
||||
if not chart_src.exists():
|
||||
chart_src = repo_root / "skills/literature-qa/chart-reading"
|
||||
chart_dst = vault / skill_dir / "literature-qa/chart-reading"
|
||||
if chart_src.exists() and chart_src.is_dir():
|
||||
for f in chart_src.glob("*.md"):
|
||||
|
|
@ -1144,7 +1166,7 @@ class DoneStep(StepScreen):
|
|||
yield from super().compose()
|
||||
vault_config = getattr(self.app, 'vault_config', {})
|
||||
system_dir = vault_config.get('system_dir', '99_System')
|
||||
worker_cmd = f"python {system_dir}/PaperForge/worker/scripts/literature_pipeline.py --vault ."
|
||||
worker_cmd = f"python -m paperforge sync --vault ."
|
||||
yield Markdown(f"""
|
||||
## 安装完成!
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue