obsidian-plugin/lllin000_PaperForge
1
0
Fork 0
mirror of https://github.com/lllin000/PaperForge.git synced 2026-07-22 06:50:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(17-dead-code-precommit): remove delegation wrappers and dead code, run ruff cleanup

Browse source 
- Remove def load_vault_config delegation wrappers from all 7 worker modules
- Add from paperforge.config import load_vault_config, paperforge_paths at module level
- Remove unused system_dir/resources_dir/control_dir extraction in pipeline_paths
- Fix sync.py intra-function imports (run_selection_sync, run_index_refresh)
- Fix repair.py import path for _resolve_formal_note_path (direct from _utils)
- Add ruff and pre-commit as test dependencies in pyproject.toml
- Configure per-file ruff ignores for pre-existing issues
- Run ruff check --fix + ruff format (353 + 58 issues auto-fixed)
- All 203 tests pass, ruff check zero warnings
This commit is contained in:
Research Assistant 2026-04-27 17:40:47 +08:00
parent c08f86de0f
commit 7cccf4eefc
11 changed files with 2216 additions and 2009 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
paperforge/ocr_diagnostics.py
View file

@ -167,7 +167,7 @@ def ocr_doctor(config: dict[str, str] | None, live: bool = False) -> dict:
return {
"level": 4,
"passed": False,
"error": f'Live PDF test failed: {poll_data.get("errorMsg", "unknown")}',
"error": f"Live PDF test failed: {poll_data.get('errorMsg', 'unknown')}",
"fix": "PDF may be unreadable or OCR service error. Try a different PDF or check service status.",
}
else:

230
paperforge/worker/base_views.py
View file

@ -1,41 +1,27 @@
from __future__ import annotations
import logging
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.config import paperforge_paths
from paperforge.worker._utils import (
read_json,
slugify_filename,
write_json,
)
logger = logging.getLogger(__name__)
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
for raw_line in env_path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split('=', 1)
key, value = line.split("=", 1)
key = key.strip()
if not key or key in os.environ:
continue
@ -44,104 +30,6 @@ def load_simple_env(env_path: Path) -> None:
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:：]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
@ -149,14 +37,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
shared = paperforge_paths(vault)
root = shared["paperforge"]
control_root = shared["control"]
@ -183,17 +64,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
config_path = paths["config"]
config = read_json(config_path) if config_path.exists() else {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
for export_path in sorted(paths["exports"].glob("*.json")):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
@ -204,14 +83,17 @@ def load_domain_config(paths: dict[str, Path]) -> dict:
write_json(config_path, config)
return config
def base_markdown_filter(path: Path, vault: Path) -> str:
try:
return str(path.relative_to(vault)).replace('\\', '/')
return str(path.relative_to(vault)).replace("\\", "/")
except ValueError:
return str(path).replace('\\', '/')
return str(path).replace("\\", "/")
PAPERFORGE_VIEW_PREFIX = "# PAPERFORGE_VIEW: "
def build_base_views(domain: str) -> list[dict]:
"""Build the 8-view list for a domain Base file.
@ -224,12 +106,33 @@ def build_base_views(domain: str) -> list[dict]:
return [
{
"name": "控制面板",
"order": ["file.name", "title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
"order": [
"file.name",
"title",
"year",
"has_pdf",
"do_ocr",
"analyze",
"ocr_status",
"deep_reading_status",
"pdf_path",
"fulltext_md_path",
],
"filter": None,
},
{
"name": "推荐分析",
"order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
"order": [
"year",
"title",
"has_pdf",
"do_ocr",
"analyze",
"ocr_status",
"deep_reading_status",
"pdf_path",
"fulltext_md_path",
],
"filter": "analyze = true AND recommend_analyze = true",
},
{
@ -259,11 +162,22 @@ def build_base_views(domain: str) -> list[dict]:
},
{
"name": "全记录",
"order": ["title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"],
"order": [
"title",
"year",
"has_pdf",
"do_ocr",
"analyze",
"ocr_status",
"deep_reading_status",
"pdf_path",
"fulltext_md_path",
],
"filter": None,
},
]
def substitute_config_placeholders(content: str, paths: dict[str, Path]) -> str:
"""Replace ${SCREAMING_SNAKE_CASE} path placeholders with vault-relative paths.
@ -297,13 +211,13 @@ def _render_views_section(views: list[dict]) -> str:
"""Render a list of view dicts to YAML views: section."""
lines = []
for v in views:
lines.append(f" - type: table")
lines.append(" - type: table")
lines.append(f' name: "{v["name"]}"')
lines.append(f" order:")
lines.append(" order:")
for col in v["order"]:
lines.append(f" - {col}")
if v["filter"]:
lines.append(f' filter: \'{v["filter"]}\'')
lines.append(f" filter: '{v['filter']}'")
return "\n".join(lines)
@ -351,7 +265,7 @@ def merge_base_views(existing_content: str | None, new_views: list[dict]) -> str
fresh_views_yaml = _render_views_section(new_views)
return f"""filters:
and:
- file.inFolder("{new_views[0]['name']}")
- file.inFolder("{new_views[0]["name"]}")
{PROPERTIES_YAML}
views:
{fresh_views_yaml}"""
@ -372,20 +286,20 @@ views:
views:
{fresh_views_yaml}"""
header_lines = lines[:views_start_idx + 1]
header_lines = lines[: views_start_idx + 1]
new_pf_blocks = []
for v in new_views:
rendered = f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n"
rendered += f" - type: table\n"
rendered += " - type: table\n"
rendered += f' name: "{v["name"]}"\n'
rendered += f" order:\n"
rendered += " order:\n"
for col in v["order"]:
rendered += f" - {col}\n"
if v["filter"]:
rendered += f' filter: \'{v["filter"]}\'\n'
rendered += f" filter: '{v['filter']}'\n"
else:
rendered += '\n'
rendered += "\n"
new_pf_blocks.append((v["name"], rendered))
rebuilt_views_lines = []
@ -396,7 +310,7 @@ views:
line = lines[i]
if line.startswith(PAPERFORGE_VIEW_PREFIX):
pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX):].strip()
pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX) :].strip()
pf_names_seen.add(pending_pf_view_name)
i += 1
continue
@ -440,16 +354,16 @@ def _build_base_yaml(folder_filter: str, views: list[dict]) -> str:
views_yaml = ""
for v in views:
views_yaml += f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n"
views_yaml += f" - type: table\n"
views_yaml += " - type: table\n"
views_yaml += f' name: "{v["name"]}"\n'
views_yaml += f" order:\n"
views_yaml += " order:\n"
for col in v["order"]:
views_yaml += f" - {col}\n"
if v["filter"]:
views_yaml += f' filter: \'{v["filter"]}\'\n'
views_yaml += f" filter: '{v['filter']}'\n"
else:
views_yaml += '\n'
views_yaml = views_yaml.rstrip('\n')
views_yaml += "\n"
views_yaml = views_yaml.rstrip("\n")
return f"""filters:
and:
- file.inFolder("{folder_filter}")
@ -515,5 +429,3 @@ def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force:
all_views = build_base_views("All Records")
pf_base = paths["bases"] / "PaperForge.base"
refresh_base(pf_base, "${LIBRARY_RECORDS}", all_views)

186
paperforge/worker/deep_reading.py
View file

@ -1,59 +1,21 @@
from __future__ import annotations
import logging
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.worker.sync import has_deep_reading_content, load_export_rows
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.ocr import validate_ocr_meta
from paperforge.config import paperforge_paths
from paperforge.worker._utils import (
STANDARD_VIEW_NAMES,
_extract_year,
_JOURNAL_DB,
load_journal_db,
lookup_impact_factor,
read_json,
read_jsonl,
slugify_filename,
write_json,
write_jsonl,
yaml_block,
yaml_list,
yaml_quote,
scan_library_records,
_resolve_formal_note_path,
write_json,
yaml_quote,
)
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.sync import has_deep_reading_content
logger = logging.getLogger(__name__)
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
@ -61,14 +23,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
shared = paperforge_paths(vault)
root = shared["paperforge"]
control_root = shared["control"]
@ -95,17 +50,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
config_path = paths["config"]
config = read_json(config_path) if config_path.exists() else {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
for export_path in sorted(paths["exports"].glob("*.json")):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
@ -116,8 +69,10 @@ def load_domain_config(paths: dict[str, Path]) -> dict:
write_json(config_path, config)
return config
# Re-exported from _utils.py for backward compatibility
def run_deep_reading(vault: Path, verbose: bool = False) -> int:
"""Sync deep-reading status between formal notes and library records.
@ -131,83 +86,104 @@ def run_deep_reading(vault: Path, verbose: bool = False) -> int:
paths = pipeline_paths(vault)
config = load_domain_config(paths)
ensure_base_views(vault, paths, config)
domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']}
{entry["export_file"]: entry["domain"] for entry in config["domains"]}
synced = 0
pending_queue: list[dict] = []
records = scan_library_records(vault)
for record in records:
key = record['zotero_key']
domain = record['domain']
key = record["zotero_key"]
domain = record["domain"]
# Status sync: check actual note content vs frontmatter status
note_path = record['note_path']
note_path = record["note_path"]
has_content = False
if note_path and note_path.exists():
note_text = note_path.read_text(encoding='utf-8')
note_text = note_path.read_text(encoding="utf-8")
has_content = has_deep_reading_content(note_text)
correct_status = 'done' if has_content else 'pending'
correct_status = "done" if has_content else "pending"
if record['deep_reading_status'] != correct_status:
record_dir = paths['library_records'] / domain
record_path = record_dir / f'{key}.md'
record_text = record_path.read_text(encoding='utf-8')
if record["deep_reading_status"] != correct_status:
record_dir = paths["library_records"] / domain
record_path = record_dir / f"{key}.md"
record_text = record_path.read_text(encoding="utf-8")
new_text = re.sub(
'^deep_reading_status:\\s*"?.*?"?$',
f'deep_reading_status: {yaml_quote(correct_status)}',
record_text, flags=re.MULTILINE, count=1
f"deep_reading_status: {yaml_quote(correct_status)}",
record_text,
flags=re.MULTILINE,
count=1,
)
record_path.write_text(new_text, encoding='utf-8')
record_path.write_text(new_text, encoding="utf-8")
synced += 1
if correct_status == 'pending':
pending_queue.append({
'zotero_key': key,
'domain': domain,
'title': record['title'],
'ocr_status': record['ocr_status'],
'is_analyze': True,
'is_do_ocr': record['do_ocr'],
})
if correct_status == "pending":
pending_queue.append(
{
"zotero_key": key,
"domain": domain,
"title": record["title"],
"ocr_status": record["ocr_status"],
"is_analyze": True,
"is_do_ocr": record["do_ocr"],
}
)
if pending_queue:
ready = [q for q in pending_queue if q['ocr_status'] == 'done']
waiting = [q for q in pending_queue if q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing')]
blocked = [q for q in pending_queue if q['is_analyze'] and q['ocr_status'] not in ('done', '') and not (q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing'))]
report_lines = ['# 待精读队列', '']
ready = [q for q in pending_queue if q["ocr_status"] == "done"]
waiting = [q for q in pending_queue if q["is_do_ocr"] and q["ocr_status"] in ("pending", "processing")]
blocked = [
q
for q in pending_queue
if q["is_analyze"]
and q["ocr_status"] not in ("done", "")
and not (q["is_do_ocr"] and q["ocr_status"] in ("pending", "processing"))
]
report_lines = ["# 待精读队列", ""]
if ready:
report_lines.extend([f'## 就绪 ({len(ready)} 篇) — OCR 已完成，可直接 /pf-deep', ''])
report_lines.extend([f"## 就绪 ({len(ready)} 篇) — OCR 已完成，可直接 /pf-deep", ""])
for q in ready:
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']}")
report_lines.append('')
report_lines.append("")
if waiting:
report_lines.extend([f'## 等待 OCR ({len(waiting)} 篇)', ''])
report_lines.extend([f"## 等待 OCR ({len(waiting)} 篇)", ""])
for q in waiting:
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status']}")
report_lines.append('')
report_lines.append("")
if blocked:
report_lines.extend([f'## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR', ''])
report_lines.extend([f"## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR", ""])
for q in blocked:
report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}")
report_lines.append('')
report_lines.append(
f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}"
)
report_lines.append("")
if verbose:
report_lines.append('### 修复步骤\n')
report_lines.append("### 修复步骤\n")
for q in blocked:
ocr_s = q['ocr_status'] or ''
if not ocr_s or ocr_s == 'pending':
fix = f"paperforge ocr"
ocr_s = q["ocr_status"] or ""
if not ocr_s or ocr_s == "pending":
fix = "paperforge ocr"
report_lines.append(f"- `{q['zotero_key']}`: 运行 `{fix}` 启动 OCR")
elif ocr_s == 'processing':
elif ocr_s == "processing":
report_lines.append(f"- `{q['zotero_key']}`: OCR 进行中，请等待完成")
elif ocr_s == 'failed':
report_lines.append(f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息，然后重新运行 `paperforge ocr`")
elif ocr_s == "failed":
report_lines.append(
f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息，然后重新运行 `paperforge ocr`"
)
else:
report_lines.append(f"- `{q['zotero_key']}`: 运行 `paperforge ocr` 重试")
report_lines.append('')
report_lines.extend(['## 操作', '', '- 对就绪论文，使用 `/pf-deep <zotero_key>` 触发精读', '- 批量触发：提供多个 key，用 subagent 并行处理', ''])
report_lines.append("")
report_lines.extend(
[
"## 操作",
"",
"- 对就绪论文，使用 `/pf-deep <zotero_key>` 触发精读",
"- 批量触发：提供多个 key，用 subagent 并行处理",
"",
]
)
else:
report_lines = ['# 待精读队列', '', '所有 analyze=true 的论文已完成精读。', '']
report_path = paths['pipeline'] / 'deep-reading-queue.md'
report_path.write_text('\n'.join(report_lines), encoding='utf-8')
print(f'deep-reading: synced {synced} records, {len(pending_queue)} pending')
report_lines = ["# 待精读队列", "", "所有 analyze=true 的论文已完成精读。", ""]
report_path = paths["pipeline"] / "deep-reading-queue.md"
report_path.write_text("\n".join(report_lines), encoding="utf-8")
print(f"deep-reading: synced {synced} records, {len(pending_queue)} pending")
return 0

1156
paperforge/worker/ocr.py
View file

File diff suppressed because it is too large Load diff

292
paperforge/worker/repair.py
View file

@ -1,155 +1,24 @@
from __future__ import annotations
import logging
import argparse
import csv
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
import logging
import re
from pathlib import Path
from paperforge.config import load_vault_config, paperforge_paths
from paperforge.worker._utils import (
_resolve_formal_note_path,
read_json,
write_json,
)
from paperforge.worker.ocr import validate_ocr_meta
from paperforge.worker.sync import (
load_export_rows,
obsidian_wikilink_for_pdf,
update_frontmatter_field,
)
from paperforge.worker.deep_reading import _resolve_formal_note_path
from paperforge.worker.ocr import validate_ocr_meta
logger = logging.getLogger(__name__)
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:：]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
@ -157,14 +26,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
shared = paperforge_paths(vault)
root = shared["paperforge"]
control_root = shared["control"]
@ -191,17 +53,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
config_path = paths["config"]
config = read_json(config_path) if config_path.exists() else {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
for export_path in sorted(paths["exports"].glob("*.json")):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
@ -212,6 +72,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict:
write_json(config_path, config)
return config
def _find_export_for_domain(paths: dict[str, Path], domain: str) -> Path | None:
"""Find the BBT export JSON file for a given domain."""
for export_path in sorted(paths["exports"].glob("*.json")):
@ -313,16 +174,10 @@ def repair_pdf_paths(
if item:
pdf_path = item.get("pdf_path", "")
if pdf_path:
new_wikilink = obsidian_wikilink_for_pdf(
pdf_path, vault, zotero_dir
)
new_wikilink = obsidian_wikilink_for_pdf(pdf_path, vault, zotero_dir)
if new_wikilink:
new_text = update_frontmatter_field(
text, "pdf_path", new_wikilink
)
new_text = update_frontmatter_field(
new_text, "path_error", ""
)
new_text = update_frontmatter_field(text, "pdf_path", new_wikilink)
new_text = update_frontmatter_field(new_text, "path_error", "")
new_text = update_frontmatter_field(
new_text,
"bbt_path_raw",
@ -380,33 +235,32 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals
"""
result = {"scanned": 0, "divergent": [], "fixed": 0, "errors": []}
config = load_domain_config(paths)
domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']}
record_paths = list(paths['library_records'].rglob('*.md'))
{entry["export_file"]: entry["domain"] for entry in config["domains"]}
record_paths = list(paths["library_records"].rglob("*.md"))
for record_path in record_paths:
try:
record_text = record_path.read_text(encoding='utf-8')
record_text = record_path.read_text(encoding="utf-8")
except Exception as e:
result['errors'].append({"file": str(record_path), "error": str(e)})
result["errors"].append({"file": str(record_path), "error": str(e)})
continue
key_match = re.search('^zotero_key:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE)
if not key_match:
continue
zotero_key = key_match.group(1).strip()
domain = record_path.parent.name
record_dir = record_path.parent
result['scanned'] += 1
result["scanned"] += 1
lib_ocr_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE)
lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else 'pending'
lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else "pending"
note_path = _resolve_formal_note_path(vault, zotero_key, domain)
note_ocr_status = None
if note_path and note_path.exists():
try:
note_text = note_path.read_text(encoding='utf-8')
note_text = note_path.read_text(encoding="utf-8")
note_status_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', note_text, re.MULTILINE)
note_ocr_status = note_status_match.group(1).strip() if note_status_match else None
except Exception:
pass
meta_path = paths['ocr'] / zotero_key / 'meta.json'
meta_path = paths["ocr"] / zotero_key / "meta.json"
meta_ocr_status = None
meta_validated_status = None
validated_status = None
@ -418,25 +272,30 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals
meta_validated_status = validated_status
if validated_error and verbose:
logger.warning("%s meta validation error: %s", zotero_key, validated_error)
raw_status = str(meta.get('ocr_status', '') or '').strip().lower()
raw_status = str(meta.get("ocr_status", "") or "").strip().lower()
meta_ocr_status = raw_status if raw_status else None
if meta_validated_status == 'done_incomplete':
meta_ocr_status = 'done_incomplete'
if meta_validated_status == "done_incomplete":
meta_ocr_status = "done_incomplete"
except Exception as e:
result['errors'].append({"file": str(meta_path), "error": str(e)})
result["errors"].append({"file": str(meta_path), "error": str(e)})
meta_ocr_status = None
is_divergent = False
div_reason = ""
if meta_validated_status == 'done_incomplete':
if meta_validated_status == "done_incomplete":
is_divergent = True
div_reason = f"meta validation: done_incomplete ({validated_error})"
elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing', None):
elif lib_ocr_status == "done" and meta_ocr_status in ("pending", "processing", None):
is_divergent = True
div_reason = f"library_record done but meta {meta_ocr_status or 'missing'}"
elif note_ocr_status == 'done' and (meta_ocr_status is None or meta_validated_status == 'done_incomplete'):
elif note_ocr_status == "done" and (meta_ocr_status is None or meta_validated_status == "done_incomplete"):
is_divergent = True
div_reason = "formal_note done but meta.json missing/invalid"
elif lib_ocr_status != 'pending' and meta_ocr_status is not None and meta_validated_status is not None and lib_ocr_status != meta_validated_status:
elif (
lib_ocr_status != "pending"
and meta_ocr_status is not None
and meta_validated_status is not None
and lib_ocr_status != meta_validated_status
):
is_divergent = True
div_reason = f"library_record={lib_ocr_status} vs meta post-validation={meta_validated_status}"
if is_divergent:
@ -448,92 +307,86 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals
"meta_ocr_status": meta_validated_status or meta_ocr_status,
"reason": div_reason,
}
result['divergent'].append(item)
result["divergent"].append(item)
if verbose:
logger.info("divergent: %s | %s", zotero_key, div_reason)
if fix:
fixed_library_record = False
fixed_formal_note = False
fixed_meta = False
new_status = 'pending'
if meta_ocr_status is None or meta_validated_status == 'done_incomplete':
new_status = 'pending'
new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status)
new_status = "pending"
if meta_ocr_status is None or meta_validated_status == "done_incomplete":
new_status = "pending"
new_record_text = update_frontmatter_field(record_text, "ocr_status", new_status)
if new_record_text != record_text:
record_path.write_text(new_record_text, encoding='utf-8')
record_path.write_text(new_record_text, encoding="utf-8")
fixed_library_record = True
if note_path and note_path.exists():
try:
note_text = note_path.read_text(encoding='utf-8')
new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status)
note_text = note_path.read_text(encoding="utf-8")
new_note_text = update_frontmatter_field(note_text, "ocr_status", new_status)
if new_note_text != note_text:
note_path.write_text(new_note_text, encoding='utf-8')
note_path.write_text(new_note_text, encoding="utf-8")
fixed_formal_note = True
except Exception:
pass
if meta_validated_status is not None and meta_validated_status != 'done':
if meta_validated_status is not None and meta_validated_status != "done":
if meta_path.exists():
try:
meta = read_json(meta_path)
meta['ocr_status'] = 'pending'
meta["ocr_status"] = "pending"
write_json(meta_path, meta)
fixed_meta = True
except Exception:
pass
record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE)
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true'
record_do_ocr_match = re.search(r"^do_ocr:\s*(true|false)$", new_record_text, re.MULTILINE)
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == "true"
if not is_do_ocr:
final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true')
final_record_text = update_frontmatter_field(new_record_text, "do_ocr", "true")
if final_record_text != new_record_text:
record_path.write_text(final_record_text, encoding='utf-8')
record_path.write_text(final_record_text, encoding="utf-8")
fixed_library_record = True
elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing'):
new_status = 'pending'
new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status)
elif lib_ocr_status == "done" and meta_ocr_status in ("pending", "processing"):
new_status = "pending"
new_record_text = update_frontmatter_field(record_text, "ocr_status", new_status)
if new_record_text != record_text:
record_path.write_text(new_record_text, encoding='utf-8')
record_path.write_text(new_record_text, encoding="utf-8")
fixed_library_record = True
if note_path and note_path.exists():
try:
note_text = note_path.read_text(encoding='utf-8')
new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status)
note_text = note_path.read_text(encoding="utf-8")
new_note_text = update_frontmatter_field(note_text, "ocr_status", new_status)
if new_note_text != note_text:
note_path.write_text(new_note_text, encoding='utf-8')
note_path.write_text(new_note_text, encoding="utf-8")
fixed_formal_note = True
except Exception:
pass
if meta_path.exists():
try:
meta = read_json(meta_path)
meta['ocr_status'] = 'pending'
meta["ocr_status"] = "pending"
write_json(meta_path, meta)
fixed_meta = True
except Exception:
pass
record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE)
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true'
record_do_ocr_match = re.search(r"^do_ocr:\s*(true|false)$", new_record_text, re.MULTILINE)
is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == "true"
if not is_do_ocr:
final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true')
final_record_text = update_frontmatter_field(new_record_text, "do_ocr", "true")
if final_record_text != new_record_text:
record_path.write_text(final_record_text, encoding='utf-8')
record_path.write_text(final_record_text, encoding="utf-8")
fixed_library_record = True
fixed_count = sum([fixed_library_record, fixed_formal_note, fixed_meta])
result['fixed'] += fixed_count
result["fixed"] += fixed_count
if verbose and fixed_count > 0:
logger.info("fixed %d files for %s", fixed_count, zotero_key)
# Path error detection and repair
path_errors = _detect_path_errors(paths, verbose)
if path_errors["total"] > 0:
error_summary = ", ".join(
f"{count} {err}" for err, count in sorted(path_errors["by_type"].items())
)
print(
f"[repair] Found {path_errors['total']} items with path errors: {error_summary}"
)
error_summary = ", ".join(f"{count} {err}" for err, count in sorted(path_errors["by_type"].items()))
print(f"[repair] Found {path_errors['total']} items with path errors: {error_summary}")
if fix_paths:
fixed_count = repair_pdf_paths(
vault, paths, path_errors["records"], verbose
)
fixed_count = repair_pdf_paths(vault, paths, path_errors["records"], verbose)
print(f"[repair] Fixed {fixed_count} PDF paths")
else:
print("[repair] Tip: run with --fix-paths to attempt auto-resolution")
@ -542,4 +395,3 @@ def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = Fals
result["path_errors"] = path_errors
return result

252
paperforge/worker/status.py
View file

@ -1,149 +1,22 @@
from __future__ import annotations
import logging
import argparse
import csv
import hashlib
import html
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.config import load_vault_config, paperforge_paths
from paperforge.worker._utils import (
read_json,
write_json,
)
from paperforge.worker.base_views import ensure_base_views
logger = logging.getLogger(__name__)
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:：]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
@ -151,14 +24,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
shared = paperforge_paths(vault)
root = shared["paperforge"]
control_root = shared["control"]
@ -185,17 +51,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
config_path = paths["config"]
config = read_json(config_path) if config_path.exists() else {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
for export_path in sorted(paths["exports"].glob("*.json")):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
@ -206,6 +70,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict:
write_json(config_path, config)
return config
def _detect_zotero_data_dir() -> str | None:
"""Try to detect the user's Zotero data directory."""
if os.name == "nt":
@ -252,7 +117,7 @@ def check_zotero_location(vault: Path, cfg: dict, add_check) -> None:
"Path Resolution",
"warn",
f"Zotero junction target missing: {target}",
f"Recreate junction: mklink /J \"{zotero_link}\" \"<Zotero数据目录>\"",
f'Recreate junction: mklink /J "{zotero_link}" "<Zotero数据目录>"',
)
except Exception:
add_check(
@ -308,9 +173,7 @@ def check_pdf_paths(vault: Path, paths: dict, add_check) -> None:
return
import random
sample = (
record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5)
)
sample = record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5)
valid = 0
errors: list[str] = []
for record_path in sample:
@ -329,9 +192,7 @@ def check_pdf_paths(vault: Path, paths: dict, add_check) -> None:
if candidate.exists():
valid += 1
else:
err_match = re.search(
r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE
)
err_match = re.search(r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE)
error_type = err_match.group(1) if err_match else "not_found"
errors.append(error_type)
total = len(sample)
@ -395,7 +256,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
if sys_version.major >= 3 and sys_version.minor >= 10:
add_check("Python 环境", "pass", f"Python {sys_version.major}.{sys_version.minor}.{sys_version.micro}")
else:
add_check("Python 环境", "fail", f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)", "升级 Python 到 3.10 或更高版本")
add_check(
"Python 环境",
"fail",
f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)",
"升级 Python 到 3.10 或更高版本",
)
required_modules = ["requests", "pymupdf", "PIL", "yaml"]
missing_modules = []
@ -405,7 +271,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
except ImportError:
missing_modules.append(mod)
if missing_modules:
add_check("Python 环境", "fail", f"缺少模块: {', '.join(missing_modules)}", f"运行: pip install {' '.join(missing_modules)}")
add_check(
"Python 环境",
"fail",
f"缺少模块: {', '.join(missing_modules)}",
f"运行: pip install {' '.join(missing_modules)}",
)
elif sys_version.major >= 3 and sys_version.minor >= 10:
pass
@ -424,13 +295,20 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
if resources_dir.exists():
add_check("Vault 结构", "pass", f"resources_dir 存在: {cfg['resources_dir']}")
else:
add_check("Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}")
add_check(
"Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}"
)
control_dir = resources_dir / cfg.get("control_dir", "LiteratureControl")
if control_dir.exists():
add_check("Vault 结构", "pass", f"control_dir 存在: {cfg.get('control_dir', 'LiteratureControl')}")
else:
add_check("Vault 结构", "fail", f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}", f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}")
add_check(
"Vault 结构",
"fail",
f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}",
f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}",
)
zotero_link = system_dir / "Zotero"
if zotero_link.exists():
@ -439,14 +317,16 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
else:
add_check("Zotero 链接", "warn", "Zotero 目录存在但不是 junction，建议使用 junction")
else:
add_check("Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} <Zotero数据目录>")
add_check(
"Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} <Zotero数据目录>"
)
exports_dir = system_dir / "PaperForge" / "exports"
if exports_dir.exists():
add_check("BBT 导出", "pass", "exports 目录存在")
else:
add_check("BBT 导出", "fail", "exports 目录不存在", "在 Better BibTeX 设置中配置导出路径")
# Check for any valid JSON export (per-domain or library.json)
json_files = sorted(exports_dir.glob("*.json")) if exports_dir.exists() else []
valid_exports = []
@ -463,7 +343,7 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
valid_exports.append((jf.name, len(data["items"])))
except (JSONDecodeError, Exception):
pass
if valid_exports:
for name, count in valid_exports:
add_check("BBT 导出", "pass", f"{name} 正常 ({count} 条)")
@ -472,7 +352,9 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
else:
add_check("BBT 导出", "fail", "未找到 JSON 导出文件", "在 Zotero Better BibTeX 设置中配置导出路径")
env_api_key = os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN")
env_api_key = (
os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN")
)
if env_api_key:
add_check("OCR 配置", "pass", "API Token 已配置")
else:
@ -480,10 +362,11 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
# Worker module check (v1.3+: pipeline/ removed, use paperforge.worker package)
try:
from paperforge.worker.sync import run_selection_sync, run_index_refresh
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.deep_reading import run_deep_reading
from paperforge.worker.ocr import run_ocr
from paperforge.worker.base_views import ensure_base_views
from paperforge.worker.sync import run_index_refresh, run_selection_sync
add_check("Worker 脚本", "pass", "paperforge.worker 包可导入")
except ImportError as e:
add_check("Worker 脚本", "fail", f"worker 函数导入失败: {e}", "运行: pip install -e .")
@ -504,6 +387,7 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
if ld_deep_script and ld_deep_script.exists():
try:
import importlib.util
spec = importlib.util.spec_from_file_location("ld_deep", ld_deep_script)
if spec and spec.loader:
mod = importlib.util.module_from_spec(spec)
@ -514,7 +398,12 @@ def run_doctor(vault: Path, verbose: bool = False) -> int:
if ld_deep_import_ok:
add_check("Agent 脚本", "pass", "paperforge and ld_deep importable")
else:
add_check("Agent 脚本", "warn", f"literature-qa skill 目录存在但 import 失败: {import_error}", "确认 agent_config_dir 配置正确并已运行 pip install -e .")
add_check(
"Agent 脚本",
"warn",
f"literature-qa skill 目录存在但 import 失败: {import_error}",
"确认 agent_config_dir 配置正确并已运行 pip install -e .",
)
else:
add_check("Agent 脚本", "warn", "literature-qa skill 目录未找到", "确认 agent_config_dir 配置正确")
@ -549,7 +438,7 @@ def _is_junction(path: Path) -> bool:
try:
import ctypes
from ctypes import wintypes
FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
GetFileAttributesW = ctypes.windll.kernel32.GetFileAttributesW
@ -569,36 +458,36 @@ def run_status(vault: Path, verbose: bool = False) -> int:
cfg = load_vault_config(vault)
config = load_domain_config(paths)
ensure_base_views(vault, paths, config)
export_files = sorted(paths['exports'].glob('*.json'))
record_count = sum(1 for _ in paths['library_records'].rglob('*.md')) if paths['library_records'].exists() else 0
note_count = sum(1 for _ in paths['literature'].rglob('*.md')) if paths['literature'].exists() else 0
base_count = sum(1 for _ in paths['bases'].glob('*.base')) if paths['bases'].exists() else 0
export_files = sorted(paths["exports"].glob("*.json"))
record_count = sum(1 for _ in paths["library_records"].rglob("*.md")) if paths["library_records"].exists() else 0
note_count = sum(1 for _ in paths["literature"].rglob("*.md")) if paths["literature"].exists() else 0
base_count = sum(1 for _ in paths["bases"].glob("*.base")) if paths["bases"].exists() else 0
ocr_done = 0
ocr_total = 0
if paths['ocr'].exists():
for meta_path in paths['ocr'].glob('*/meta.json'):
if paths["ocr"].exists():
for meta_path in paths["ocr"].glob("*/meta.json"):
ocr_total += 1
try:
meta = read_json(meta_path)
except Exception:
continue
if str(meta.get('ocr_status', '')).strip().lower() == 'done':
if str(meta.get("ocr_status", "")).strip().lower() == "done":
ocr_done += 1
env_paths = [vault / '.env', paths['pipeline'] / '.env']
env_found = [str(path.relative_to(vault)).replace('\\', '/') for path in env_paths if path.exists()]
env_paths = [vault / ".env", paths["pipeline"] / ".env"]
env_found = [str(path.relative_to(vault)).replace("\\", "/") for path in env_paths if path.exists()]
# Count path errors
path_error_count = 0
if paths['library_records'].exists():
for record_path in paths['library_records'].rglob('*.md'):
if paths["library_records"].exists():
for record_path in paths["library_records"].rglob("*.md"):
try:
text = record_path.read_text(encoding='utf-8')
text = record_path.read_text(encoding="utf-8")
if re.search(r'^path_error:\s*"(.+?)"\s*$', text, re.MULTILINE):
path_error_count += 1
except Exception:
continue
print('PaperForge Lite status')
print("PaperForge Lite status")
print(f"- vault: {vault}")
print(f"- system_dir: {cfg['system_dir']}")
print(f"- resources_dir: {cfg['resources_dir']}")
@ -616,6 +505,7 @@ def run_status(vault: Path, verbose: bool = False) -> int:
print(f"- env: {', '.join(env_found) if env_found else 'not configured'}")
return 0
# =============================================================================
# Update 功能
# =============================================================================
@ -624,5 +514,3 @@ GITHUB_REPO = "LLLin000/PaperForge"
GITHUB_ZIP = f"https://github.com/{GITHUB_REPO}/archive/refs/heads/master.zip"
UPDATEABLE_PATHS = ["skills", "pipeline", "command", "scripts"]

1814
paperforge/worker/sync.py
View file

File diff suppressed because it is too large Load diff

196
paperforge/worker/update.py
View file

@ -1,147 +1,26 @@
from __future__ import annotations
import logging
import argparse
import csv
import hashlib
import html
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.parse
from json import JSONDecodeError
import zipfile
from datetime import datetime, timezone
from datetime import datetime
from pathlib import Path
from xml.etree import ElementTree as ET
import requests
import fitz
from PIL import Image
from paperforge.config import load_vault_config, paperforge_paths
from paperforge.worker._utils import (
read_json,
write_json,
)
logger = logging.getLogger(__name__)
STANDARD_VIEW_NAMES = frozenset([
"控制面板", "推荐分析", "待 OCR", "OCR 完成",
"待深度阅读", "深度阅读完成", "正式卡片", "全记录"
])
def load_simple_env(env_path: Path) -> None:
if not env_path.exists():
return
for raw_line in env_path.read_text(encoding='utf-8').splitlines():
line = raw_line.strip()
if not line or line.startswith('#') or '=' not in line:
continue
key, value = line.split('=', 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}):
value = value[1:-1]
os.environ[key] = value
def read_json(path: Path):
return json.loads(path.read_text(encoding='utf-8'))
_JOURNAL_DB: dict[str, dict] | None = None
def load_journal_db(vault: Path) -> dict[str, dict]:
"""Load zoterostyle.json journal database."""
global _JOURNAL_DB
if _JOURNAL_DB is not None:
return _JOURNAL_DB
zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json'
if zoterostyle_path.exists():
try:
_JOURNAL_DB = read_json(zoterostyle_path)
except (JSONDecodeError, Exception):
_JOURNAL_DB = {}
else:
_JOURNAL_DB = {}
return _JOURNAL_DB
def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str:
"""Lookup impact factor: prefer zoterostyle.json, fallback to extra field."""
if not journal_name:
return ''
journal_db = load_journal_db(vault)
if journal_name in journal_db:
rank_data = journal_db[journal_name].get('rank', {})
if isinstance(rank_data, dict):
sciif = rank_data.get('sciif', '')
if sciif:
return str(sciif)
if extra:
if_match = re.search('影响因子[:：]\\s*([0-9.]+)', extra)
if if_match:
return if_match.group(1)
return ''
def write_json(path: Path, data) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
def read_jsonl(path: Path):
rows = []
if not path.exists():
return rows
for line in path.read_text(encoding='utf-8').splitlines():
line = line.strip()
if line:
rows.append(json.loads(line))
return rows
def write_jsonl(path: Path, rows) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows))
if text:
text += '\n'
path.write_text(text, encoding='utf-8')
def yaml_quote(value: str) -> str:
if isinstance(value, bool):
return 'true' if value else 'false'
return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"'
def yaml_block(value: str) -> list[str]:
value = (value or '').strip()
if not value:
return ['abstract: |-', ' ']
lines = ['abstract: |-']
for line in value.splitlines():
lines.append(f' {line}')
return lines
def yaml_list(key: str, values) -> list[str]:
cleaned = [str(value).strip() for value in values or [] if str(value).strip()]
if not cleaned:
return [f'{key}: []']
lines = [f'{key}:']
for value in cleaned:
lines.append(f' - {yaml_quote(value)}')
return lines
def slugify_filename(text: str) -> str:
cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip()
return cleaned[:120] or 'untitled'
def _extract_year(value: str) -> str:
match = re.search('(19|20)\\d{2}', value or '')
return match.group(0) if match else ''
def load_vault_config(vault: Path) -> dict:
"""Read vault configuration — delegates to shared resolver.
Preserves the public name for legacy callers. Configuration precedence:
1. paperforge.config.load_vault_config (overrides > env > JSON > defaults)
"""
from paperforge.config import load_vault_config as _shared_load_vault_config
return _shared_load_vault_config(vault)
def pipeline_paths(vault: Path) -> dict[str, Path]:
"""Build complete PaperForge path inventory — delegates to shared resolver.
@ -149,14 +28,7 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
Returns paths from paperforge.config.paperforge_paths() plus
worker-only keys. Preserves all legacy keys for existing callers.
"""
from paperforge.config import paperforge_paths as _shared_paperforge_paths
shared = _shared_paperforge_paths(vault)
cfg = load_vault_config(vault)
system_dir = cfg["system_dir"]
resources_dir = cfg["resources_dir"]
control_dir = cfg["control_dir"]
shared = paperforge_paths(vault)
root = shared["paperforge"]
control_root = shared["control"]
@ -183,17 +55,15 @@ def pipeline_paths(vault: Path) -> dict[str, Path]:
"ocr_queue": root / "ocr" / "ocr-queue.json",
}
def load_domain_config(paths: dict[str, Path]) -> dict:
"""Load or create the Lite domain mapping from export JSON files."""
config_path = paths['config']
if config_path.exists():
config = read_json(config_path)
else:
config = {"domains": []}
config_path = paths["config"]
config = read_json(config_path) if config_path.exists() else {"domains": []}
domains = config.setdefault("domains", [])
known_exports = {str(entry.get("export_file", "")) for entry in domains}
changed = not config_path.exists()
for export_path in sorted(paths['exports'].glob('*.json')):
for export_path in sorted(paths["exports"].glob("*.json")):
if export_path.name in known_exports:
continue
domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []})
@ -204,6 +74,7 @@ def load_domain_config(paths: dict[str, Path]) -> dict:
write_json(config_path, config)
return config
def protected_paths(vault: Path) -> set[str]:
cfg = load_vault_config(vault)
pf = f"{cfg['system_dir']}/PaperForge"
@ -219,11 +90,12 @@ def protected_paths(vault: Path) -> set[str]:
}
def _remote_version() -> str | None:
try:
api = f"https://api.github.com/repos/{GITHUB_REPO}/contents/paperforge.json"
req = urllib.request.Request(api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"})
req = urllib.request.Request(
api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"}
)
with urllib.request.urlopen(req, timeout=10) as resp:
data = json.loads(resp.read())
req2 = urllib.request.Request(data["download_url"], headers={"User-Agent": "PaperForge"})
@ -260,7 +132,7 @@ def _do_backup(vault: Path, updates: list) -> Path | None:
backup_dir = vault / f".backup_{datetime.now():%Y%m%d_%H%M%S}"
backup_dir.mkdir(exist_ok=True)
count = 0
for src, dst, action in updates:
for _src, dst, action in updates:
if action == "UPDATE" and dst.exists():
bp = backup_dir / dst.relative_to(vault)
bp.parent.mkdir(parents=True, exist_ok=True)
@ -273,7 +145,7 @@ def _do_backup(vault: Path, updates: list) -> Path | None:
def _apply_updates(vault: Path, updates: list) -> bool:
try:
for src, dst, action in updates:
for src, dst, _action in updates:
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
return True
@ -295,21 +167,22 @@ def _rollback(vault: Path, backup_dir: Path) -> None:
def _detect_install_method() -> tuple[str, Path | None]:
"""Detect how paperforge is installed."""
import paperforge
pkg_dir = Path(paperforge.__file__).parent.resolve()
# Check if installed in site-packages (pip install)
if "site-packages" in str(pkg_dir) or "dist-packages" in str(pkg_dir):
return ("pip", pkg_dir)
# Check if in editable mode (pip install -e .)
if pkg_dir.name == "paperforge" and (pkg_dir.parent / ".git").exists():
return ("pip-editable", pkg_dir.parent)
# Check if vault has .git (git clone)
vault = Path.cwd()
if (vault / ".git").exists():
return ("git", vault)
return ("unknown", None)
@ -321,8 +194,8 @@ def _update_via_pip(editable: bool = False) -> bool:
else:
cmd.append("--upgrade")
cmd.append("paperforge")
logger.info("执行: %s", ' '.join(cmd))
logger.info("执行: %s", " ".join(cmd))
r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8")
if r.returncode != 0:
logger.error("pip 更新失败: %s", r.stderr)
@ -373,7 +246,7 @@ def update_via_zip(vault: Path) -> bool:
logger.info("所有文件已是最新")
return True
logger.info("发现 %d 个文件需要更新:", len(updates))
for src, dst, action in updates:
for _src, dst, action in updates:
logger.info(" [%s] %s", action, dst.relative_to(vault))
backup = _do_backup(vault, updates)
if _apply_updates(vault, updates):
@ -401,12 +274,14 @@ def run_update(vault: Path) -> int:
logger.info("PaperForge Lite 更新")
logger.info("%s", "=" * 50)
logger.info("本地版本: %s", local)
logger.info("远程版本: %s", remote or 'unknown')
logger.info("远程版本: %s", remote or "unknown")
if not remote:
logger.error("无法获取远程版本")
return 1
try:
needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple(int(x) for x in local.split(".") if x.isdigit())
needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple(
int(x) for x in local.split(".") if x.isdigit()
)
except ValueError:
needs = remote != local
if not needs:
@ -418,11 +293,11 @@ def run_update(vault: Path) -> int:
if ans not in ("y", "yes"):
logger.info("已取消")
return 0
# Auto-detect installation method
method, path = _detect_install_method()
logger.info("安装方式: %s", method)
if method == "pip":
logger.info("通过 pip 更新...")
success = _update_via_pip(editable=False)
@ -443,7 +318,7 @@ def run_update(vault: Path) -> int:
else:
logger.warning("未检测到标准安装方式，尝试 zip 下载...")
success = _update_via_zip(vault)
if success:
logger.info("更新完成！请重启 Obsidian")
return 0 if success else 1
@ -452,4 +327,3 @@ def run_update(vault: Path) -> int:
# =============================================================================
# Main
# =============================================================================

20
pyproject.toml
View file

@ -36,6 +36,7 @@ dependencies = [
[project.optional-dependencies]
test = [
"pytest>=7.4.0",
"ruff>=0.4.0",
]
[project.scripts]
@ -54,10 +55,29 @@ addopts = "--ignore=tests/sandbox/00_TestVault/"
[tool.ruff]
target-version = "py310"
line-length = 120
exclude = ["paperforge/skills"]
[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "SIM"]
ignore = ["B904"] # raise-within-try is intentional in many worker error paths
[tool.ruff.lint.per-file-ignores]
# Pre-existing: undefined names in update.py (GITHUB_REPO etc. defined below function defs)
"paperforge/worker/update.py" = ["F821"]
# Pre-existing: undefined names in sync.py (run_search_sources, run_ingest_candidates)
"paperforge/worker/sync.py" = ["F821", "B007", "E501"]
# Pre-existing: style suggestions not part of dead code removal
"paperforge/worker/ocr.py" = ["SIM102", "E501", "B905"]
"paperforge/worker/repair.py" = ["SIM102"]
"paperforge/ocr_diagnostics.py" = ["E501", "SIM105"]
# Pre-existing: importability check in status.py uses try/except imports
"paperforge/worker/status.py" = ["F401"]
# Pre-existing: test files use patterns incompatible with strict ruff rules
"tests/**" = ["E501", "E402", "SIM117", "SIM108", "F841", "B007", "SIM103", "SIM105", "SIM222", "B905"]
# Pre-existing: scripts with formatting not part of this cleanup
"scripts/consistency_audit.py" = ["SIM110", "SIM102"]
"scripts/welcome.py" = ["E501"]
"scripts/validate_setup.py" = ["E501"]
[tool.ruff.format]
quote-style = "double"

43
scripts/consistency_audit.py
View file

@ -12,11 +12,10 @@ Exit code: 0 if all pass, 1 if any fail.
from __future__ import annotations
import os
import re
import sys
from collections.abc import Iterable
from pathlib import Path
from typing import Iterable
# Root of the repository (parent of this script)
REPO_ROOT = Path(__file__).parent.parent.resolve()
@ -77,10 +76,7 @@ COMMAND_DOC_REQUIRED_SECTIONS = [
def should_exclude_check1(path: Path) -> bool:
"""Check if a path should be excluded from Check 1."""
rel = path.relative_to(REPO_ROOT).as_posix()
for exclude in CHECK1_EXCLUDES:
if rel.startswith(exclude) or exclude in rel:
return True
return False
return any(rel.startswith(exclude) or exclude in rel for exclude in CHECK1_EXCLUDES)
def find_files(pattern: str, root: Path = REPO_ROOT) -> Iterable[Path]:
@ -111,7 +107,7 @@ def check_old_commands() -> tuple[int, list[str]]:
line_end = len(content)
line = content[line_start:line_end]
# Allow old commands in explicit migration sections of AGENTS.md
if rel == "AGENTS.md" and "命令迁移说明" in content[:match.start()]:
if rel == "AGENTS.md" and "命令迁移说明" in content[: match.start()]:
# Check if this is in section 11 (migration section)
section_start = content.find("## 11. 命令迁移说明")
if section_start != -1 and match.start() > section_start:
@ -203,9 +199,7 @@ def check_dead_links() -> tuple[int, list[str]]:
if not target_path.exists():
# Try appending .md for implicit markdown links
if not (target_path.with_suffix(".md")).exists():
violations.append(
f" [{rel}] Dead link to '{link_target}' (text: '{link_text}')"
)
violations.append(f" [{rel}] Dead link to '{link_target}' (text: '{link_text}')")
passed = len(violations) == 0
return (0 if passed else 1), violations
@ -243,6 +237,7 @@ def check_command_docs() -> tuple[int, list[str]]:
def check_duplicate_utils() -> tuple[int, list[str]]:
"""Check 5: Detect worker modules with duplicated utility functions outside the re-export from _utils.py."""
import ast
violations: list[str] = []
worker_dir = REPO_ROOT / "paperforge" / "worker"
@ -250,13 +245,23 @@ def check_duplicate_utils() -> tuple[int, list[str]]:
return 0, violations
# Known shared function names from _utils.py
shared_funcs = frozenset({
"read_json", "write_json", "read_jsonl", "write_jsonl",
"yaml_quote", "yaml_block", "yaml_list",
"slugify_filename", "_extract_year",
"load_journal_db", "lookup_impact_factor",
"scan_library_records", "_resolve_formal_note_path",
})
shared_funcs = frozenset(
{
"read_json",
"write_json",
"read_jsonl",
"write_jsonl",
"yaml_quote",
"yaml_block",
"yaml_list",
"slugify_filename",
"_extract_year",
"load_journal_db",
"lookup_impact_factor",
"scan_library_records",
"_resolve_formal_note_path",
}
)
skip_files = frozenset({"_utils.py", "_retry.py", "_progress.py", "__init__.py"})
@ -293,9 +298,7 @@ def check_duplicate_utils() -> tuple[int, list[str]]:
if is_reexport:
continue
violations.append(
f" [{rel}] Function '{node.name}' is defined locally but exists in _utils.py"
)
violations.append(f" [{rel}] Function '{node.name}' is defined locally but exists in _utils.py")
passed = len(violations) == 0
return (0 if passed else 1), violations

34
tests/test_legacy_worker_compat.py
View file

@ -1,4 +1,5 @@
"""Test compatibility: worker modules use shared resolver."""
from __future__ import annotations
import json
@ -22,7 +23,7 @@ def tmp_vault(tmp_path: Path) -> Path:
(paperforge / "candidates").mkdir(parents=True)
resources = tmp_path / "03_Resources"
literature = resources / "Literature"
resources / "Literature"
control = resources / "LiteratureControl"
(control / "library-records").mkdir(parents=True)
@ -58,13 +59,13 @@ class TestWorkerLoadVaultConfig:
shared_cfg = shared_load(tmp_vault)
worker_cfg = worker_load(tmp_vault)
assert set(worker_cfg.keys()) == set(shared_cfg.keys()), (
f"Key mismatch: worker={set(worker_cfg.keys())} vs shared={set(shared_cfg.keys())}"
)
assert set(worker_cfg.keys()) == set(
shared_cfg.keys()
), f"Key mismatch: worker={set(worker_cfg.keys())} vs shared={set(shared_cfg.keys())}"
for key in shared_cfg:
assert worker_cfg.get(key) == shared_cfg.get(key), (
f"Key '{key}' differs: worker={worker_cfg.get(key)!r} vs shared={shared_cfg.get(key)!r}"
)
assert worker_cfg.get(key) == shared_cfg.get(
key
), f"Key '{key}' differs: worker={worker_cfg.get(key)!r} vs shared={shared_cfg.get(key)!r}"
def test_nested_vault_config_respected(self, tmp_vault: Path) -> None:
"""Nested vault_config block takes precedence over top-level keys."""
@ -126,9 +127,7 @@ class TestWorkerPipelinePaths:
for key in expected_keys:
assert key in paths, f"Missing key: {key}"
def test_pipeline_paths_values_from_shared_resolver(
self, tmp_vault: Path
) -> None:
def test_pipeline_paths_values_from_shared_resolver(self, tmp_vault: Path) -> None:
"""Verify shared resolver keys have correct values."""
from paperforge.config import paperforge_paths as shared_paths
from paperforge.worker.sync import pipeline_paths
@ -137,9 +136,7 @@ class TestWorkerPipelinePaths:
worker = pipeline_paths(tmp_vault)
for key in ["exports", "ocr", "library_records", "literature", "control", "bases"]:
assert worker[key] == shared[key], (
f"Key '{key}' differs: worker={worker[key]} vs shared={shared[key]}"
)
assert worker[key] == shared[key], f"Key '{key}' differs: worker={worker[key]} vs shared={shared[key]}"
class TestLegacyStatusSubprocess:
@ -159,10 +156,7 @@ class TestLegacyStatusSubprocess:
env=env,
)
# Should exit 0, no import errors
assert result.returncode == 0, (
f"status command failed with code {result.returncode}\n"
f"stdout: {result.stdout}\nstderr: {result.stderr}"
)
assert "ImportError" not in result.stderr, (
f"Import error in stderr: {result.stderr}"
)
assert (
result.returncode == 0
), f"status command failed with code {result.returncode}\nstdout: {result.stdout}\nstderr: {result.stderr}"
assert "ImportError" not in result.stderr, f"Import error in stderr: {result.stderr}"