diff --git a/.planning/REQUIREMENTS-v1.2.md b/.planning/REQUIREMENTS-v1.2.md deleted file mode 100644 index eb3675dd..00000000 --- a/.planning/REQUIREMENTS-v1.2.md +++ /dev/null @@ -1,154 +0,0 @@ -# Requirements: v1.2 Systematization & Cohesion - -> Requirements for Milestone v1.2. Derived from PROJECT.md and architecture research. - ---- - -## Context - -Milestone v1.1 hardened the sandbox onboarding flow. v1.2 transforms the project from a functional-but-scattered prototype into a cohesive, user-centric system. - -Current state: -- Agent commands: `/LD-deep`, `/LD-paper`, `/lp-ocr`, `/lp-index-refresh`, `/lp-selection-sync`, `/lp-status` -- CLI commands: `paperforge selection-sync`, `index-refresh`, `ocr run`, `ocr doctor`, `deep-reading`, `repair`, `status` -- Two naming conventions confuse users -- `selection-sync` + `index-refresh` are almost always run together but are separate commands - ---- - -## Requirements - -### SYS-01: Unified Agent Command Namespace - -**Description:** All agent commands must use the `/pf-*` prefix consistently. - -**Acceptance Criteria:** -- [ ] `/pf-deep` replaces `/LD-deep` (deep reading) -- [ ] `/pf-paper` replaces `/LD-paper` (quick summary) -- [ ] `/pf-ocr` replaces `/lp-ocr` (OCR worker) -- [ ] `/pf-sync` replaces `/lp-selection-sync` + `/lp-index-refresh` (combined sync) -- [ ] `/pf-status` replaces `/lp-status` (status check) -- [ ] `/pf-doctor` is available (diagnostics) -- [ ] Old commands (`/LD-*`, `/lp-*`) still work but show deprecation warning -- [ ] All command docs updated to use new namespace - -**Priority:** P0 - ---- - -### SYS-02: CLI Simplification - -**Description:** Combine frequently-co-occurring CLI commands into user-centric workflows. - -**Acceptance Criteria:** -- [ ] `paperforge sync` combines `selection-sync` + `index-refresh` with sensible defaults -- [ ] `paperforge sync --full` runs both with verbose output -- [ ] `paperforge sync --dry-run` previews what would happen -- [ ] Old commands (`selection-sync`, `index-refresh`) still work but show "consider using sync" hint -- [ ] `paperforge ocr` is simplified (evaluate whether `ocr run` + `ocr doctor` should merge) -- [ ] CLI help text is user-centric, not worker-centric - -**Priority:** P0 - ---- - -### SYS-03: Command Consistency Matrix - -**Description:** Ensure 1:1 mapping between agent commands and CLI commands where appropriate. - -**Acceptance Criteria:** -- [ ] Matrix documenting agent command ↔ CLI command mapping exists -- [ ] Where functionality overlaps, names align (`/pf-sync` ↔ `paperforge sync`) -- [ ] Agent-only commands (`/pf-deep`) have clear CLI equivalents or documented rationale for absence -- [ ] CLI-only commands (`paperforge repair`) have documented agent equivalents or rationale - -**Priority:** P1 - ---- - -### SYS-04: Architecture Documentation - -**Description:** Document the intended architecture for contributors and advanced users. - -**Acceptance Criteria:** -- [ ] `docs/ARCHITECTURE.md` exists explaining the two-layer design (Worker + Agent) -- [ ] `docs/COMMANDS.md` exists with unified command reference -- [ ] Command dispatch pattern documented (how `/pf-*` maps to implementation) -- [ ] Directory structure rationale documented - -**Priority:** P1 - ---- - -### SYS-05: Migration Guide - -**Description:** Help existing users transition from old commands to new commands. - -**Acceptance Criteria:** -- [ ] `docs/MIGRATION-v1.2.md` exists -- [ ] Old → new command mapping table provided -- [ ] Deprecation timeline documented (when will `/LD-*` and `/lp-*` be removed?) -- [ ] No breaking changes to data formats or storage - -**Priority:** P1 - ---- - -### SYS-06: Command Doc Restructuring - -**Description:** Restructure command docs to match unified namespace and improve discoverability. - -**Acceptance Criteria:** -- [ ] All command docs in `command/` use `/pf-*` naming -- [ ] Each command doc includes: purpose, prerequisites, examples, related commands -- [ ] Cross-references between agent and CLI versions of same command -- [ ] AGENTS.md updated with unified command reference - -**Priority:** P1 - ---- - -### SYS-07: Test Coverage for Unified Commands - -**Description:** Ensure tests cover both old and new command names during deprecation period. - -**Acceptance Criteria:** -- [ ] Smoke tests verify `/pf-*` commands work -- [ ] Backward compatibility tests verify `/LD-*` and `/lp-*` still work -- [ ] Deprecation warnings are tested -- [ ] No regression in existing 17 smoke tests - -**Priority:** P1 - ---- - -## Out of Scope - -- No new functional features (no new AI capabilities, no new OCR features) -- No breaking changes to data formats, storage, or `.planning/` structure -- No changes to Zotero/Better BibTeX integration -- No changes to OCR provider (still PaddleOCR) -- No scheduled/automated worker triggers - ---- - -## Dependencies - -- v1.1 must be complete (Phase 8 done, all tests passing) -- All existing command docs must be readable -- `get-shit-done-main` architecture research completed - ---- - -## Success Criteria - -1. A new user can look at the command list and understand what each command does without knowing implementation details -2. Existing users can continue using old commands during deprecation period -3. All tests pass (existing 17 + new unified command tests) -4. Documentation is consistent across agent and CLI interfaces -5. The system "feels" cohesive — one namespace, one mental model - ---- - -*Created: 2026-04-24* -*Milestone: v1.2 Systematization & Cohesion* diff --git a/pipeline/__init__.py b/pipeline/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pipeline/worker/__init__.py b/pipeline/worker/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pipeline/worker/scripts/__init__.py b/pipeline/worker/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pipeline/worker/scripts/literature_pipeline.py b/pipeline/worker/scripts/literature_pipeline.py deleted file mode 100644 index 6904bb46..00000000 --- a/pipeline/worker/scripts/literature_pipeline.py +++ /dev/null @@ -1,4041 +0,0 @@ -from __future__ import annotations -import argparse -import csv -import hashlib -import html -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import urllib.parse -from json import JSONDecodeError -import zipfile -from datetime import datetime, timezone -from pathlib import Path -from xml.etree import ElementTree as ET -import requests -import fitz -from PIL import Image - -STANDARD_VIEW_NAMES = frozenset([ - "控制面板", "推荐分析", "待 OCR", "OCR 完成", - "待深度阅读", "深度阅读完成", "正式卡片", "全记录" -]) - -def load_simple_env(env_path: Path) -> None: - if not env_path.exists(): - return - for raw_line in env_path.read_text(encoding='utf-8').splitlines(): - line = raw_line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - key, value = line.split('=', 1) - key = key.strip() - if not key or key in os.environ: - continue - value = value.strip() - if len(value) >= 2 and value[0] == value[-1] and (value[0] in {'"', "'"}): - value = value[1:-1] - os.environ[key] = value - -def read_json(path: Path): - return json.loads(path.read_text(encoding='utf-8')) -_JOURNAL_DB: dict[str, dict] | None = None - -def load_journal_db(vault: Path) -> dict[str, dict]: - """Load zoterostyle.json journal database.""" - global _JOURNAL_DB - if _JOURNAL_DB is not None: - return _JOURNAL_DB - zoterostyle_path = vault / load_vault_config(vault)['system_dir'] / 'Zotero' / 'zoterostyle.json' - if zoterostyle_path.exists(): - try: - _JOURNAL_DB = read_json(zoterostyle_path) - except (JSONDecodeError, Exception): - _JOURNAL_DB = {} - else: - _JOURNAL_DB = {} - return _JOURNAL_DB - -def lookup_impact_factor(journal_name: str, extra: str, vault: Path) -> str: - """Lookup impact factor: prefer zoterostyle.json, fallback to extra field.""" - if not journal_name: - return '' - journal_db = load_journal_db(vault) - if journal_name in journal_db: - rank_data = journal_db[journal_name].get('rank', {}) - if isinstance(rank_data, dict): - sciif = rank_data.get('sciif', '') - if sciif: - return str(sciif) - if extra: - if_match = re.search('影响因子[:：]\\s*([0-9.]+)', extra) - if if_match: - return if_match.group(1) - return '' - -def write_json(path: Path, data) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') - -def read_jsonl(path: Path): - rows = [] - if not path.exists(): - return rows - for line in path.read_text(encoding='utf-8').splitlines(): - line = line.strip() - if line: - rows.append(json.loads(line)) - return rows - -def write_jsonl(path: Path, rows) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - text = '\n'.join((json.dumps(row, ensure_ascii=False) for row in rows)) - if text: - text += '\n' - path.write_text(text, encoding='utf-8') - -def yaml_quote(value: str) -> str: - if isinstance(value, bool): - return 'true' if value else 'false' - return '"' + str(value or '').replace('\\', '\\\\').replace('"', '\\"') + '"' - -def yaml_block(value: str) -> list[str]: - value = (value or '').strip() - if not value: - return ['abstract: |-', ' '] - lines = ['abstract: |-'] - for line in value.splitlines(): - lines.append(f' {line}') - return lines - -def yaml_list(key: str, values) -> list[str]: - cleaned = [str(value).strip() for value in values or [] if str(value).strip()] - if not cleaned: - return [f'{key}: []'] - lines = [f'{key}:'] - for value in cleaned: - lines.append(f' - {yaml_quote(value)}') - return lines - -def slugify_filename(text: str) -> str: - cleaned = re.sub('[<>:"/\\\\|?*]+', '', text).strip() - return cleaned[:120] or 'untitled' - -def _extract_year(value: str) -> str: - match = re.search('(19|20)\\d{2}', value or '') - return match.group(0) if match else '' - - -def load_vault_config(vault: Path) -> dict: - """Read vault configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. Configuration precedence: - 1. paperforge.config.load_vault_config (overrides > env > JSON > defaults) - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - - -def pipeline_paths(vault: Path) -> dict[str, Path]: - """Build complete PaperForge path inventory — delegates to shared resolver. - - Returns paths from paperforge.config.paperforge_paths() plus - worker-only keys. Preserves all legacy keys for existing callers. - """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - - cfg = load_vault_config(vault) - system_dir = cfg["system_dir"] - resources_dir = cfg["resources_dir"] - control_dir = cfg["control_dir"] - - root = shared["paperforge"] - control_root = shared["control"] - - return { - **shared, - # Worker-only keys (added on top of shared resolver output) - "pipeline": root, - "candidates": root / "candidates" / "candidates.json", - "candidate_inbox": root / "candidates" / "inbox", - "candidate_archive": root / "candidates" / "archive", - "search_tasks": root / "search" / "tasks", - "search_archive": root / "search" / "archive", - "search_results": root / "search" / "results", - "harvest_root": root / "skill-prototypes" / "zotero-review-manuscript-writer", - "records": control_root / "candidate-records", - "review": root / "candidates" / "review-latest.md", - "config": root / "config" / "domain-collections.json", - "queue": root / "writeback" / "writeback-queue.jsonl", - "log": root / "writeback" / "writeback-log.jsonl", - "bridge_config": root / "zotero-bridge" / "bridge-config.json", - "bridge_config_sample": root / "zotero-bridge" / "bridge-config.sample.json", - "index": root / "indexes" / "formal-library.json", - "ocr_queue": root / "ocr" / "ocr-queue.json", - } - -def load_domain_config(paths: dict[str, Path]) -> dict: - """Load or create the Lite domain mapping from export JSON files.""" - config_path = paths['config'] - if config_path.exists(): - config = read_json(config_path) - else: - config = {"domains": []} - domains = config.setdefault("domains", []) - known_exports = {str(entry.get("export_file", "")) for entry in domains} - changed = not config_path.exists() - for export_path in sorted(paths['exports'].glob('*.json')): - if export_path.name in known_exports: - continue - domains.append({"domain": export_path.stem, "export_file": export_path.name, "allowed_collections": []}) - known_exports.add(export_path.name) - changed = True - if changed: - config_path.parent.mkdir(parents=True, exist_ok=True) - write_json(config_path, config) - return config - -def base_markdown_filter(path: Path, vault: Path) -> str: - try: - return str(path.relative_to(vault)).replace('\\', '/') - except ValueError: - return str(path).replace('\\', '/') - -PAPERFORGE_VIEW_PREFIX = "# PAPERFORGE_VIEW: " - -def build_base_views(domain: str) -> list[dict]: - """Build the 8-view list for a domain Base file. - - Args: - domain: The domain name (e.g., "骨科") — passed for compatibility but each view has fixed name/filter. - - Returns: - List of 8 view dicts, each with keys: name (str), order (list), filter (str|None). - """ - return [ - { - "name": "控制面板", - "order": ["file.name", "title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"], - "filter": None, - }, - { - "name": "推荐分析", - "order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"], - "filter": "analyze = true AND recommend_analyze = true", - }, - { - "name": "待 OCR", - "order": ["year", "title", "has_pdf", "do_ocr", "ocr_status", "pdf_path"], - "filter": 'do_ocr = true AND ocr_status = "pending"', - }, - { - "name": "OCR 完成", - "order": ["year", "title", "has_pdf", "do_ocr", "ocr_status", "pdf_path"], - "filter": 'ocr_status = "done"', - }, - { - "name": "待深度阅读", - "order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"], - "filter": 'analyze = true AND ocr_status = "done" AND deep_reading_status = "pending"', - }, - { - "name": "深度阅读完成", - "order": ["year", "title", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path"], - "filter": 'deep_reading_status = "done"', - }, - { - "name": "正式卡片", - "order": ["title", "year", "has_pdf", "deep_reading_status", "pdf_path"], - "filter": 'deep_reading_status = "done"', - }, - { - "name": "全记录", - "order": ["title", "year", "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", "pdf_path", "fulltext_md_path"], - "filter": None, - }, - ] - -def substitute_config_placeholders(content: str, paths: dict[str, Path]) -> str: - """Replace ${SCREAMING_SNAKE_CASE} path placeholders with vault-relative paths. - - Args: - content: The Base file content string with ${PLACEHOLDER} tokens. - paths: dict of path key -> Path objects (from paperforge_paths()). - - Returns: - Content with placeholders replaced by vault-relative paths. - Unrecognized placeholders are left unchanged. - """ - substitutions = { - "LIBRARY_RECORDS": paths.get("library_records"), - "LITERATURE": paths.get("literature"), - "CONTROL_DIR": paths.get("control"), - } - result = content - for key, path in substitutions.items(): - if path is not None: - vault = paths.get("vault") - if vault is not None: - try: - rel = path.relative_to(vault) - result = result.replace(f"${{{key}}}", str(rel).replace(chr(92), "/")) - except ValueError: - result = result.replace(f"${{{key}}}", str(path).replace(chr(92), "/")) - return result - - -def _render_views_section(views: list[dict]) -> str: - """Render a list of view dicts to YAML views: section.""" - lines = [] - for v in views: - lines.append(f" - type: table") - lines.append(f' name: "{v["name"]}"') - lines.append(f" order:") - for col in v["order"]: - lines.append(f" - {col}") - if v["filter"]: - lines.append(f' filter: \'{v["filter"]}\'') - return "\n".join(lines) - - -def merge_base_views(existing_content: str | None, new_views: list[dict]) -> str: - """Incrementally merge standard PaperForge views into an existing .base file. - - Strategy: - - PaperForge generates exactly 8 views with known names (STANDARD_VIEW_NAMES). - - Any OTHER views in the existing file are user-defined and MUST be preserved. - - Each PaperForge view is preceded by a PAPERFORGE_VIEW_PREFIX comment marker. - - On refresh: replace ALL PaperForge views (identified by prefix) with fresh ones. - - User views (no prefix) are left completely untouched. - - Args: - existing_content: Raw text of existing .base file (or None/empty for fresh generation). - new_views: List of 8 view dicts from build_base_views(). - - Returns: - Merged .base file content with PaperForge views updated, user views preserved. - """ - PROPERTIES_YAML = """properties: - zotero_key: - displayName: "Zotero Key" - title: - displayName: "Title" - year: - displayName: "Year" - has_pdf: - displayName: "PDF" - do_ocr: - displayName: "OCR" - analyze: - displayName: "Analyze" - ocr_status: - displayName: "OCR Status" - deep_reading_status: - displayName: "Deep Reading" - pdf_path: - displayName: "PDF Path" - fulltext_md_path: - displayName: "Fulltext" -""" - - if not existing_content or not existing_content.strip(): - fresh_views_yaml = _render_views_section(new_views) - return f"""filters: - and: - - file.inFolder("{new_views[0]['name']}") -{PROPERTIES_YAML} -views: -{fresh_views_yaml}""" - - lines = existing_content.split("\n") - views_start_idx = None - for i, line in enumerate(lines): - if line.strip().startswith("views:"): - views_start_idx = i - break - - if views_start_idx is None: - fresh_views_yaml = _render_views_section(new_views) - return f"""{existing_content} - -# --- PaperForge views regenerated (views: section was missing) --- -{PROPERTIES_YAML} -views: -{fresh_views_yaml}""" - - header_lines = lines[:views_start_idx + 1] - - new_pf_blocks = [] - for v in new_views: - rendered = f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n" - rendered += f" - type: table\n" - rendered += f' name: "{v["name"]}"\n' - rendered += f" order:\n" - for col in v["order"]: - rendered += f" - {col}\n" - if v["filter"]: - rendered += f' filter: \'{v["filter"]}\'\n' - else: - rendered += '\n' - new_pf_blocks.append((v["name"], rendered)) - - rebuilt_views_lines = [] - pf_names_seen = set() - i = views_start_idx + 1 - pending_pf_view_name = None - while i < len(lines): - line = lines[i] - - if line.startswith(PAPERFORGE_VIEW_PREFIX): - pending_pf_view_name = line[len(PAPERFORGE_VIEW_PREFIX):].strip() - pf_names_seen.add(pending_pf_view_name) - i += 1 - continue - elif pending_pf_view_name is not None and line.strip().startswith("- type: table"): - pf_block = next((b for n, b in new_pf_blocks if n == pending_pf_view_name), None) - if pf_block: - rebuilt_views_lines.append(pf_block) - pending_pf_view_name = None - i += 1 - continue - elif line.strip().startswith("- type: table"): - pending_pf_view_name = None - view_block_lines = [line] - i += 1 - while i < len(lines): - next_line = lines[i] - if next_line.strip().startswith(PAPERFORGE_VIEW_PREFIX): - break - if next_line.strip().startswith("- type: table"): - break - if next_line.strip() and not next_line.startswith(" ") and not next_line.startswith("\t"): - break - view_block_lines.append(next_line) - i += 1 - rebuilt_views_lines.append("\n".join(view_block_lines)) - continue - else: - pending_pf_view_name = None - i += 1 - - for view_name, pf_block in new_pf_blocks: - if view_name not in pf_names_seen: - rebuilt_views_lines.append(pf_block) - - result_lines = header_lines + rebuilt_views_lines - return "\n".join(result_lines) - - -def _build_base_yaml(folder_filter: str, views: list[dict]) -> str: - """Build complete .base YAML with PAPERFORGE_VIEW_PREFIX markers on each view.""" - views_yaml = "" - for v in views: - views_yaml += f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n" - views_yaml += f" - type: table\n" - views_yaml += f' name: "{v["name"]}"\n' - views_yaml += f" order:\n" - for col in v["order"]: - views_yaml += f" - {col}\n" - if v["filter"]: - views_yaml += f' filter: \'{v["filter"]}\'\n' - else: - views_yaml += '\n' - views_yaml = views_yaml.rstrip('\n') - return f"""filters: - and: - - file.inFolder("{folder_filter}") -properties: - zotero_key: - displayName: "Zotero Key" - title: - displayName: "Title" - year: - displayName: "Year" - has_pdf: - displayName: "PDF" - do_ocr: - displayName: "OCR" - analyze: - displayName: "Analyze" - ocr_status: - displayName: "OCR Status" - deep_reading_status: - displayName: "Deep Reading" - pdf_path: - displayName: "PDF Path" - fulltext_md_path: - displayName: "Fulltext" -views: -{views_yaml}""" - - -def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: bool = False) -> None: - """Generate/refresh Domain Base files with incremental merge (preserves user views). - - Each PaperForge standard view is marked with PAPERFORGE_VIEW_PREFIX. On refresh, - only PaperForge views are replaced; user-defined views are preserved. - """ - paths["bases"].mkdir(parents=True, exist_ok=True) - - def refresh_base(base_path: Path, folder_filter: str, views: list[dict]) -> None: - """Refresh a single .base file: merge PaperForge views, preserve user views.""" - if base_path.exists() and not force: - existing = base_path.read_text(encoding="utf-8") - merged = merge_base_views(existing, views) - else: - merged = _build_base_yaml(folder_filter, views) - merged = substitute_config_placeholders(merged, paths) - base_path.write_text(merged, encoding="utf-8") - - seen_domains = set() - for entry in config.get("domains", []): - domain = str(entry.get("domain", "") or "").strip() - if not domain or domain in seen_domains: - continue - seen_domains.add(domain) - - domain_views = build_base_views(domain) - folder_filter = f"${{LIBRARY_RECORDS}}/{domain}" - base_path = paths["bases"] / f"{slugify_filename(domain)}.base" - refresh_base(base_path, folder_filter, domain_views) - - hub_views = build_base_views("Literature Hub") - hub_path = paths["bases"] / "Literature Hub.base" - refresh_base(hub_path, "${LIBRARY_RECORDS}", hub_views) - - all_views = build_base_views("All Records") - pf_base = paths["bases"] / "PaperForge.base" - refresh_base(pf_base, "${LIBRARY_RECORDS}", all_views) - - -def build_collection_lookup(collections: dict) -> dict: - path_cache = {} - item_paths = {} - - def path_for(key: str) -> str: - if key in path_cache: - return path_cache[key] - node = collections.get(key, {}) - parent = node.get('parent') or '' - name = node.get('name', '') - parent_path = path_for(parent) if parent else '' - full_path = f'{parent_path}/{name}' if parent_path else name - path_cache[key] = full_path - return full_path - for key, node in collections.items(): - full_path = path_for(key) - for item_id in node.get('items', []): - item_paths.setdefault(item_id, []).append(full_path) - return {'path_by_key': path_cache, 'paths_by_item_id': item_paths} - -def export_collection_paths(export_path: Path) -> list[str]: - data = read_json(export_path) - if not isinstance(data, dict): - return [] - collections = data.get('collections', {}) - if not isinstance(collections, dict): - return [] - lookup = build_collection_lookup(collections) - paths = sorted({path for path in lookup.get('path_by_key', {}).values() if str(path or '').strip()}, key=lambda value: (value.count('/'), value)) - return paths - -def load_domain_collection_catalog(paths: dict[str, Path]) -> dict[str, list[str]]: - config_path = paths['config'] - config = read_json(config_path) if config_path.exists() else {'domains': []} - domain_entries = list(config.get('domains', [])) - entry_by_export = {entry.get('export_file', ''): dict(entry) for entry in domain_entries if entry.get('export_file')} - export_files = sorted(paths['exports'].glob('*.json')) - changed = False - for export_path in export_files: - entry = entry_by_export.get(export_path.name) - if not entry: - entry = {'domain': export_path.stem, 'export_file': export_path.name, 'allowed_collections': []} - entry_by_export[export_path.name] = entry - changed = True - derived = export_collection_paths(export_path) - if entry.get('allowed_collections', []) != derived: - entry['allowed_collections'] = derived - changed = True - domains = sorted(entry_by_export.values(), key=lambda entry: entry.get('domain', '')) - if changed or config.get('domains', []) != domains: - write_json(config_path, {'domains': domains}) - return {entry.get('domain', ''): entry.get('allowed_collections', []) for entry in domains if entry.get('domain')} - -def load_export_inventory(paths: dict[str, Path]) -> dict[str, dict]: - inventory = {'doi': {}, 'pmid': {}, 'title': {}} - for export_path in sorted(paths['exports'].glob('*.json')): - domain = export_path.stem - for item in load_export_rows(export_path): - record = {'zotero_key': item.get('key', ''), 'domain': domain, 'title': item.get('title', ''), 'doi': item.get('doi', ''), 'pmid': item.get('pmid', ''), 'collections': item.get('collections', [])} - doi = str(record.get('doi', '') or '').strip().lower() - pmid = str(record.get('pmid', '') or '').strip() - title = normalize_candidate_title(record.get('title', '')) - if doi and doi not in inventory['doi']: - inventory['doi'][doi] = record - if pmid and pmid not in inventory['pmid']: - inventory['pmid'][pmid] = record - if title and title not in inventory['title']: - inventory['title'][title] = record - return inventory - -def find_existing_library_match(row: dict, inventory: dict[str, dict]) -> dict | None: - doi = str(row.get('doi', '') or '').strip().lower() - if doi and doi in inventory['doi']: - return inventory['doi'][doi] - pmid = str(row.get('pmid', '') or '').strip() - if pmid and pmid in inventory['pmid']: - return inventory['pmid'][pmid] - title = normalize_candidate_title(row.get('title', '')) - if title and title in inventory['title']: - return inventory['title'][title] - return None - -def resolve_collection_choice(domain: str, raw_value: str, catalog: dict[str, list[str]]) -> dict[str, str]: - text = str(raw_value or '').strip() - if not text: - return {'resolved': '', 'match': '', 'input': ''} - allowed = [path for path in catalog.get(domain, []) if path] - if not allowed: - return {'resolved': '', 'match': 'no_catalog', 'input': text} - lower_text = text.lower() - if '/' not in text: - leaf_matches = [path for path in allowed if path.split('/')[-1].strip().lower() == lower_text] - leaf_matches = sorted(set(leaf_matches)) - if len(leaf_matches) > 1: - return {'resolved': '', 'match': 'ambiguous_leaf', 'input': text} - exact_map = {path: path for path in allowed} - if text in exact_map: - return {'resolved': text, 'match': 'exact', 'input': text} - lower_exact = {path.lower(): path for path in allowed} - if lower_text in lower_exact: - return {'resolved': lower_exact[lower_text], 'match': 'exact_ci', 'input': text} - leaf_matches = [path for path in allowed if path.split('/')[-1].strip().lower() == lower_text] - if len(leaf_matches) == 1: - return {'resolved': leaf_matches[0], 'match': 'leaf', 'input': text} - suffix_matches = [path for path in allowed if path.lower().endswith('/' + lower_text) or path.lower() == lower_text] - suffix_matches = sorted(set(suffix_matches)) - if len(suffix_matches) == 1: - return {'resolved': suffix_matches[0], 'match': 'suffix', 'input': text} - compact = re.sub('\\s+', '', lower_text) - compact_matches = [] - for path in allowed: - path_compact = re.sub('\\s+', '', path.lower()) - if path_compact.endswith('/' + compact) or path_compact == compact: - compact_matches.append(path) - compact_matches = sorted(set(compact_matches)) - if len(compact_matches) == 1: - return {'resolved': compact_matches[0], 'match': 'compact_suffix', 'input': text} - match = 'ambiguous' if leaf_matches or suffix_matches or compact_matches else 'unresolved' - return {'resolved': '', 'match': match, 'input': text} - -def apply_candidate_collection_resolution(row: dict, catalog: dict[str, list[str]]) -> dict: - resolved = dict(row) - domain = str(resolved.get('domain', '') or '').strip() - recommended = resolve_collection_choice(domain, resolved.get('recommended_collection', ''), catalog) - user = resolve_collection_choice(domain, resolved.get('user_collection', ''), catalog) - resolved['recommended_collection'] = recommended.get('resolved', '') - resolved['user_collection_resolved'] = user.get('resolved', '') - if str(resolved.get('user_collection', '') or '').strip(): - resolved['final_collection'] = user.get('resolved', '') - resolved['collection_resolution'] = f"user_{user.get('match', 'unresolved')}" if user.get('match') else 'user_unresolved' - else: - resolved['final_collection'] = recommended.get('resolved', '') - resolved['collection_resolution'] = f"recommended_{recommended.get('match', 'unresolved')}" if recommended.get('match') else 'recommended_unresolved' - return resolved - -def apply_existing_library_match(row: dict, inventory: dict[str, dict]) -> dict: - resolved = dict(row) - match = find_existing_library_match(resolved, inventory) - if not match: - resolved['existing_zotero_key'] = '' - resolved['existing_collections'] = [] - resolved['duplicate_hint'] = '' - return resolved - resolved['existing_zotero_key'] = str(match.get('zotero_key', '') or '').strip() - resolved['existing_collections'] = list(match.get('collections', []) or []) - collections_text = ' | '.join(resolved['existing_collections']) - if collections_text: - resolved['duplicate_hint'] = f"已存在于 Zotero: {resolved['existing_zotero_key']} ({collections_text})" - else: - resolved['duplicate_hint'] = f"已存在于 Zotero: {resolved['existing_zotero_key']}" - return resolved - -def resolve_item_collection_paths(item: dict, collection_lookup: dict) -> list[str]: - paths = [] - collection_keys = item.get('collections') or [] - if collection_keys: - for key in collection_keys: - paths.append(collection_lookup.get('path_by_key', {}).get(key, key)) - item_id = item.get('itemID') - if item_id is not None: - paths.extend(collection_lookup.get('paths_by_item_id', {}).get(item_id, [])) - return sorted({path for path in paths if path}, key=lambda value: (-value.count('/'), value)) - -def obsidian_wikilink_for_pdf(pdf_path: str, vault_dir: Path, zotero_dir: Path | None = None) -> str: - text = str(pdf_path or '').strip() - if not text: - return '' - # Handle storage: prefix paths by resolving through zotero_dir - if text.startswith('storage:') and zotero_dir is not None: - storage_rel = text[len('storage:'):].lstrip('/').lstrip('\\') - absolute_pdf_path = (zotero_dir / storage_rel.replace('/', os.sep)).resolve() - absolute_str = str(absolute_pdf_path) - else: - absolute_str = absolutize_vault_path(vault_dir, text, resolve_junction=True) - if not absolute_str: - return '' - absolute_path = Path(absolute_str) - try: - relative = absolute_path.relative_to(vault_dir) - except ValueError: - return f'[[{absolute_path.as_posix()}]]' - return f'[[{relative.as_posix()}]]' - -def absolutize_vault_path(vault: Path, path: str, resolve_junction: bool = False) -> str: - text = str(path or '').strip() - if not text: - return '' - candidate = Path(text) - if candidate.is_absolute(): - result = str(candidate) - else: - result = str((vault / text.replace('/', os.sep)).resolve()) - if resolve_junction: - from paperforge.pdf_resolver import resolve_junction - result = str(resolve_junction(Path(result))) - return result - -def obsidian_wikilink_for_path(vault: Path, path: str) -> str: - absolute = absolutize_vault_path(vault, path, resolve_junction=True) - if not absolute: - return '' - absolute_path = Path(absolute) - try: - relative = absolute_path.relative_to(vault) - except ValueError: - return f'[[{absolute_path.as_posix()}]]' - return f'[[{relative.as_posix()}]]' - -def collection_fields(collection_paths: list[str]) -> dict[str, str | list[str]]: - paths = [path for path in collection_paths if path] - primary = paths[0] if paths else '' - if paths: - primary = sorted(paths, key=lambda value: (value.count('/'), len(value), value), reverse=True)[0] - tags = [] - seen = set() - for path in paths: - for part in [segment.strip() for segment in path.split('/') if segment.strip()]: - if part not in seen: - seen.add(part) - tags.append(part) - group = primary - return {'collections': paths, 'collection_tags': tags, 'collection_group': [group] if group else []} - -def extract_authors(item: dict) -> list[str]: - authors = [] - for creator in item.get('creators', []): - if creator.get('creatorType') != 'author': - continue - full_name = ' '.join((part for part in [creator.get('firstName', ''), creator.get('lastName', '')] if part)).strip() - if full_name: - authors.append(full_name) - elif creator.get('name'): - authors.append(creator['name']) - return authors - -def _normalize_attachment_path(path: str, zotero_dir: Path | None = None) -> tuple[str, str, str]: - """Normalize a BBT attachment path to a consistent storage: format. - - Handles three real-world BBT export formats: - 1. Absolute Windows paths: D:\\...\\Zotero\\storage\\8CHARKEY\\filename.pdf - -> storage:8CHARKEY/filename.pdf - 2. storage: prefix: storage:KEY/filename.pdf -> pass through - 3. Bare relative: KEY/filename.pdf -> storage:KEY/filename.pdf - - Args: - path: Raw path from BBT JSON attachment. - zotero_dir: Optional absolute path to Zotero data directory for - validating absolute paths. - - Returns: - Tuple of (normalized_path, bbt_path_raw, zotero_storage_key). - normalized_path uses forward slashes and storage: prefix for - Zotero storage paths. bbt_path_raw preserves the original input - for debugging. zotero_storage_key is the 8-character Zotero key. - """ - raw = str(path or '').strip() - if not raw: - return ('', '', '') - - bbt_path_raw = raw - - # Format 2: Already has storage: prefix — pass through with slash normalization - if raw.startswith('storage:'): - storage_rel = raw[len('storage:'):].lstrip('/').lstrip('\\') - storage_rel = storage_rel.replace('\\', '/') - parts = storage_rel.split('/') - zotero_storage_key = parts[0] if parts else '' - return (f'storage:{storage_rel}', bbt_path_raw, zotero_storage_key) - - # Format 1: Absolute Windows path pointing to Zotero storage - candidate = Path(raw) - if candidate.is_absolute(): - norm_path = raw.replace('\\', '/') - # Detect Zotero storage pattern: .../storage/8CHARKEY/... - if '/storage/' in norm_path: - parts_after_storage = norm_path.split('/storage/', 1)[1] - parts = parts_after_storage.split('/') - if len(parts) >= 2 and len(parts[0]) == 8 and parts[0].isalnum(): - zotero_storage_key = parts[0] - filename = '/'.join(parts[1:]) - return (f'storage:{zotero_storage_key}/{filename}', - bbt_path_raw, zotero_storage_key) - # Absolute path but not in Zotero storage — mark as absolute - return (f'absolute:{raw}', bbt_path_raw, '') - - # Format 3: Bare relative path — prepend storage: prefix - norm = raw.replace('\\', '/') - parts = norm.split('/') - zotero_storage_key = parts[0] if parts else '' - return (f'storage:{norm}', bbt_path_raw, zotero_storage_key) - - -def _identify_main_pdf(attachments: list[dict]) -> tuple[dict | None, list[dict]]: - """Identify the main PDF and supplementary materials from attachments. - - Uses a hybrid three-priority strategy (Decision D-02): - 1. Primary: attachment.title == "PDF" AND contentType == "application/pdf" - 2. Fallback heuristic: largest file by size (if available), else shortest title - 3. Final fallback: first PDF attachment in the list - - Args: - attachments: List of attachment dicts from load_export_rows(). - - Returns: - Tuple of (main_pdf_attachment, supplementary_attachments). - main_pdf_attachment may be None if no PDFs found. - supplementary_attachments is a list of all other PDF attachments. - """ - pdf_attachments = [ - a for a in attachments - if isinstance(a, dict) and a.get('contentType') == 'application/pdf' - ] - - if not pdf_attachments: - return (None, []) - - # Priority 1: Title exactly equals "PDF" - for att in pdf_attachments: - if att.get('title') == 'PDF': - main = att - supplementary = [a for a in pdf_attachments if a is not main] - return (main, supplementary) - - # Priority 2: Largest file by size (if size field is available and differentiated) - sized = [(a, a.get('size', 0) or 0) for a in pdf_attachments] - sized.sort(key=lambda x: x[1], reverse=True) - if sized and sized[0][1] > 0: - if len(sized) == 1 or sized[0][1] > sized[1][1]: - main = sized[0][0] - supplementary = [a for a in pdf_attachments if a is not main] - return (main, supplementary) - - # Priority 2b (sizes equal or unavailable): shortest title - titled = [(a, len(str(a.get('title', '')))) for a in pdf_attachments] - titled.sort(key=lambda x: x[1]) - main = titled[0][0] - supplementary = [a for a in pdf_attachments if a is not main] - return (main, supplementary) - - -def load_export_rows(path: Path) -> list[dict]: - data = read_json(path) - if isinstance(data, list): - return data - if isinstance(data, dict) and isinstance(data.get('items'), list): - collection_lookup = build_collection_lookup(data.get('collections', {})) - rows = [] - for item in data['items']: - if item.get('itemType') in {'attachment', 'note', 'annotation'}: - continue - attachments = [] - for attachment in item.get('attachments', []): - if not isinstance(attachment, dict): - continue - raw_path = attachment.get('path', '') - normalized_path, bbt_path_raw, zotero_storage_key = _normalize_attachment_path(raw_path) - # Preserve contentType from BBT if present; fallback to file extension - content_type = attachment.get('contentType', '') - if not content_type and str(normalized_path).lower().endswith('.pdf'): - content_type = 'application/pdf' - attachments.append({ - 'path': normalized_path, - 'contentType': content_type, - 'title': attachment.get('title', ''), - 'bbt_path_raw': bbt_path_raw, - 'zotero_storage_key': zotero_storage_key, - 'size': attachment.get('size', 0) or 0, - }) - main_pdf, supplementary_pdfs = _identify_main_pdf(attachments) - pdf_path = main_pdf['path'] if main_pdf else '' - bbt_path_raw = main_pdf['bbt_path_raw'] if main_pdf else '' - zotero_storage_key = main_pdf['zotero_storage_key'] if main_pdf else '' - path_error = 'not_found' if not main_pdf else '' - supplementary = [a['path'] for a in supplementary_pdfs] if supplementary_pdfs else [] - attachment_count = len(attachments) - rows.append({ - 'key': item.get('key') or item.get('itemKey', ''), - 'title': item.get('title', ''), - 'authors': extract_authors(item), - 'abstract': item.get('abstractNote', ''), - 'journal': item.get('publicationTitle', ''), - 'year': _extract_year(item.get('date', '')), - 'date': item.get('date', ''), - 'doi': item.get('DOI', ''), - 'pmid': item.get('PMID', ''), - 'collections': resolve_item_collection_paths(item, collection_lookup), - 'attachments': attachments, - 'pdf_path': pdf_path, - 'supplementary': supplementary, - 'attachment_count': attachment_count, - 'bbt_path_raw': bbt_path_raw, - 'zotero_storage_key': zotero_storage_key, - 'path_error': path_error, - }) - return rows - raise ValueError(f'Unsupported export format: {path}') - -def compute_final_collection(row: dict) -> str: - user_raw = str(row.get('user_collection', '') or '').strip() - user_resolved = str(row.get('user_collection_resolved', '') or '').strip() - recommended = str(row.get('recommended_collection', '') or '').strip() - if user_raw: - return user_resolved - return recommended - -def canonicalize_decision(value: str) -> str: - text = str(value or '').strip() - if text in {'', '待查'}: - return '待定' - if text in {'排除', '不纳入'}: - return '不纳入' - if text == '纳入': - return '纳入' - return '待定' - -def candidate_markdown(row: dict) -> str: - row = dict(row) - row['final_collection'] = compute_final_collection(row) - row['decision'] = canonicalize_decision(row.get('decision', '')) - lines = ['---'] - ordered_keys = ['candidate_id', 'domain', 'title', 'authors', 'year', 'journal', 'doi', 'pmid', 'source', 'requester_skill', 'request_context', 'abstract_short', 'decision', 'recommended_collection', 'recommend_confidence', 'recommend_reason', 'user_collection', 'user_collection_resolved', 'final_collection', 'collection_resolution', 'duplicate_hint', 'existing_zotero_key', 'existing_collections', 'import_status', 'note', 'candidate_source_type', 'source_zotero_key', 'cited_ref_number', 'trigger_sentence', 'source_context', 'task_relevance_reason', 'harvest_priority', 'raw_reference', 'status'] - row.setdefault('status', 'candidate') - for key in ordered_keys: - value = row.get(key, '') - if isinstance(value, list): - lines.append(f'{key}:') - for item in value: - lines.append(f' - {yaml_quote(item)}') - elif value == '': - lines.append(f'{key}:') - elif '\n' in str(value): - lines.extend(yaml_block(str(value)).copy() if key == 'abstract' else [f'{key}: |-'] + [f' {line}' for line in str(value).splitlines()]) - else: - lines.append(f'{key}: {yaml_quote(value)}') - lines.extend(['---', '', f"# {row['candidate_id']}", '', '候选文献轻量记录，仅用于 Base 决策和 write-back 触发，不是正式文献卡片。', '']) - return '\n'.join(lines) - -def generate_review(candidates: list[dict]) -> str: - normalized = [] - for row in candidates: - copy = dict(row) - copy['decision'] = canonicalize_decision(copy.get('decision', '')) - normalized.append(copy) - include = [c for c in normalized if c.get('decision') == '纳入'] - exclude = [c for c in normalized if c.get('decision') == '不纳入'] - lines = ['# 本轮候选总览', '', '## 检索背景', '', f'- 候选数量：{len(normalized)}', f'- 建议纳入：{len(include)}', f'- 不纳入：{len(exclude)}', '', '## 总体判断', '', '- 当前候选池已经按决策状态分层，可直接进入 Base 处理。', '', '## 推荐优先纳入', ''] - if include: - for row in include: - lines.extend([f"### {row['candidate_id']}", '', f"- 标题：{row['title']}", f'- 推荐分类：`{compute_final_collection(row)}`', f"- 理由：{row.get('recommend_reason', '')}", '']) - else: - lines.extend(['- 暂无', '']) - lines.extend(['## 不纳入', '']) - if exclude: - for row in exclude: - lines.extend([f"### {row['candidate_id']}", '', f"- 标题：{row['title']}", f"- 理由：{row.get('recommend_reason', '')}", '']) - else: - lines.extend(['- 暂无', '']) - lines.extend(['## 下一步', '', '1. 在 Base 中确认决策。', '2. 对纳入项执行 write-back。', '3. 刷新正式索引。', '']) - return '\n'.join(lines) -DEEP_READING_HEADER = '## 🔍 精读' - -def extract_preserved_deep_reading(text: str) -> str: - """Extract the `## 🔍 精读` section by matching it as a real markdown header. - - Uses regex to ensure we match `## 🔍 精读` at the start of a line, - avoiding false positives from prose text that merely mentions the string. - """ - if not text: - return '' - match = re.search('^## 🔍 精读\\s*$', text, re.MULTILINE) - if not match: - return '' - start = match.start() - preserved = text[start:].strip() - return preserved - -def has_deep_reading_content(text: str) -> bool: - """Return True only if the deep-reading section contains *substantive* content. - - A scaffold alone (filled with placeholders like '（待补充）') does NOT count. - We strip out structural lines (section headers, callout headers, empty lists) - and placeholder text, then require at least one prose sentence or 20 chars - of actual content. - """ - preserved = extract_preserved_deep_reading(text) - if not preserved: - return False - body = preserved.replace(DEEP_READING_HEADER, '').strip() - if not body: - return False - lines = body.splitlines() - non_placeholder_chars = 0 - has_prose_sentence = False - for line in lines: - stripped = line.strip() - if not stripped: - continue - if stripped.startswith('### '): - continue - if re.match('^>\\s*\\[!', stripped): - continue - if '（待补充）' in stripped: - continue - if re.match('^[-*]\\s*$', stripped): - continue - non_placeholder_chars += len(stripped) - if re.search('[\\u4e00-\\u9fff]', stripped) and re.search('[。！？\\.\\!\\?]$', stripped): - has_prose_sentence = True - return has_prose_sentence or non_placeholder_chars >= 20 - -def library_record_markdown(row: dict) -> str: - lines = ['---', f"zotero_key: {row.get('zotero_key', '')}", f"domain: {row.get('domain', '')}", f"title: {yaml_quote(row.get('title', ''))}", f"year: {row.get('year', '')}", f"doi: {yaml_quote(row.get('doi', ''))}", f"date: {yaml_quote(row.get('date', ''))}", f"collection_path: {yaml_quote(row.get('collection_path', ''))}", f"has_pdf: {('true' if row.get('has_pdf') else 'false')}", f"pdf_path: {yaml_quote(row.get('pdf_path', ''))}", f"bbt_path_raw: {yaml_quote(row.get('bbt_path_raw', ''))}", f"zotero_storage_key: {yaml_quote(row.get('zotero_storage_key', ''))}", f"attachment_count: {row.get('attachment_count', 0)}"] - # supplementary as YAML list of wikilinks (already formatted by caller) - supplementary = row.get('supplementary', []) - if supplementary: - lines.append('supplementary:') - for wikilink in supplementary: - lines.append(f' - {yaml_quote(wikilink)}') - else: - lines.append('supplementary: []') - # path_error only emitted when there is an actual error - if row.get('path_error'): - lines.append(f"path_error: {yaml_quote(row.get('path_error', ''))}") - lines.extend([ - f"fulltext_md_path: {yaml_quote(row.get('fulltext_md_path', ''))}", - f"recommend_analyze: {('true' if row.get('recommend_analyze') else 'false')}", - f"analyze: {('true' if row.get('analyze') else 'false')}", - f"do_ocr: {('true' if row.get('do_ocr') else 'false')}", - f"ocr_status: {yaml_quote(row.get('ocr_status', 'pending'))}", - f"deep_reading_status: {yaml_quote(row.get('deep_reading_status', 'pending'))}", - f"analysis_note: {yaml_quote(row.get('analysis_note', ''))}", - ]) - lines.extend(yaml_list('collection_group', row.get('collection_group', []))) - lines.extend(yaml_list('collections', row.get('collections', []))) - lines.extend(yaml_list('collection_tags', row.get('collection_tags', []))) - lines.append(f"first_author: {yaml_quote(row.get('first_author', ''))}") - lines.append(f"journal: {yaml_quote(row.get('journal', ''))}") - lines.append(f"impact_factor: {yaml_quote(row.get('impact_factor', ''))}") - lines.extend(['---', '', f"# {row.get('title', '')}", '', '正式库控制记录。', '', '- `recommend_analyze` 仅由 `has_pdf=true` 推导。', '- `analyze` 控制是否生成正式文献卡片。', '- `do_ocr` 控制 OCR 任务。', '- `deep_reading_status` 仅两级：`pending`（未精读）/ `done`（已精读）。', '']) - return '\n'.join(lines) - -def _add_missing_frontmatter_fields(existing_content: str, new_fields: dict[str, str]) -> str: - """Surgically append missing fields to existing frontmatter without overwriting anything.""" - if not existing_content.startswith('---'): - return existing_content - parts = existing_content.split('---', 2) - if len(parts) < 3: - return existing_content - frontmatter = parts[1] - body = parts[2] - lines_to_add = [] - for key, value in new_fields.items(): - pattern = '^' + re.escape(key) + '\\s*:' - if not re.search(pattern, frontmatter, re.MULTILINE): - lines_to_add.append(f'{key}: {yaml_quote(value)}') - if not lines_to_add: - return existing_content - new_frontmatter = frontmatter.rstrip('\n') + '\n' + '\n'.join(lines_to_add) + '\n' - return f'---{new_frontmatter}---{body}' - -def update_frontmatter_field(content: str, key: str, value: str) -> str: - """Update an existing frontmatter field value, or add if missing.""" - if not content.startswith('---'): - return content - pattern = '^' + re.escape(key) + '\\s*:.*$' - replacement = f'{key}: {yaml_quote(value)}' - new_content, count = re.subn(pattern, replacement, content, flags=re.MULTILINE, count=1) - if count == 0: - new_content = _add_missing_frontmatter_fields(content, {key: value}) - return new_content - -def parse_existing_library_record(path: Path) -> dict: - if not path.exists(): - return {} - text = path.read_text(encoding='utf-8') - result = {} - for key in ('analyze', 'recommend_analyze', 'do_ocr'): - match = re.search(f'^{key}:\\s*(true|false)$', text, re.MULTILINE) - if match: - result[key] = match.group(1) == 'true' - for key in ('ocr_status', 'analysis_note'): - match = re.search(f'^{key}:\\s*"?(.*?)"?$', text, re.MULTILINE) - if match: - result[key] = match.group(1) - for key in ('deep_reading_status',): - match = re.search(f'^{key}:\\s*"?(.*?)"?$', text, re.MULTILINE) - if match: - result[key] = match.group(1) - return result - -def load_control_actions(paths: dict[str, Path]) -> dict[str, dict]: - actions = {} - if not paths['library_records'].exists(): - return actions - for record in paths['library_records'].rglob('*.md'): - text = record.read_text(encoding='utf-8') - key_match = re.search('^zotero_key:\\s*(.+)$', text, re.MULTILINE) - if not key_match: - continue - zotero_key = key_match.group(1).strip() - row = parse_existing_library_record(record) - actions[zotero_key] = {'analyze': row.get('analyze', False), 'do_ocr': row.get('do_ocr', False)} - return actions - -def run_selection_sync(vault: Path) -> int: - paths = pipeline_paths(vault) - config = load_domain_config(paths) - ensure_base_views(vault, paths, config) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} - written = 0 - updated = 0 - for export_path in sorted(paths['exports'].glob('*.json')): - domain = domain_lookup.get(export_path.name, export_path.stem) - for item in load_export_rows(export_path): - pdf_attachments = [a for a in item.get('attachments', []) if a.get('contentType') == 'application/pdf'] - has_pdf = bool(pdf_attachments) - raw_pdf_path = pdf_attachments[0].get('path', '') if pdf_attachments else '' - from paperforge.pdf_resolver import resolve_pdf_path - from paperforge.config import load_vault_config as _load_vault_config - cfg = _load_vault_config(vault) - zotero_dir = vault / cfg.get('system_dir', '99_System') / 'Zotero' - resolved_pdf = resolve_pdf_path(raw_pdf_path, has_pdf, vault, zotero_dir) - collection_meta = collection_fields(item.get('collections', [])) - record_dir = paths['library_records'] / domain - record_dir.mkdir(parents=True, exist_ok=True) - record_path = record_dir / f"{item['key']}.md" - existing = parse_existing_library_record(record_path) - meta_path = paths['ocr'] / item['key'] / 'meta.json' - meta = read_json(meta_path) if meta_path.exists() else {} - validated_ocr_status, validated_error = validate_ocr_meta(paths, meta) if meta else ('pending', '') - if meta: - meta['ocr_status'] = validated_ocr_status - if validated_error: - meta['error'] = validated_error - write_json(meta_path, meta) - note_path = paths['literature'] / domain / f"{item['key']} - {slugify_filename(item['title'])}.md" - note_text = note_path.read_text(encoding='utf-8') if note_path.exists() else '' - fulltext_md_path = obsidian_wikilink_for_path(vault, meta.get('fulltext_md_path', '') or meta.get('markdown_path', '')) - ocr_status = meta.get('ocr_status', 'pending') - if not has_pdf or not resolved_pdf: - record_ocr_status = 'nopdf' - else: - record_ocr_status = ocr_status - creators = item.get('creators', []) - first_author = '' - for c in creators: - if c.get('creatorType') == 'author': - first_author = f"{c.get('firstName', '')} {c.get('lastName', '')}".strip() - break - journal = item.get('publicationTitle', '') - extra = item.get('extra', '') - impact_factor = lookup_impact_factor(journal, extra, vault) - # Convert supplementary storage: paths to wikilinks - supplementary_wikilinks = [] - for supp_path in item.get('supplementary', []): - if supp_path: - wikilink = obsidian_wikilink_for_pdf(supp_path, vault, zotero_dir) - if wikilink: - supplementary_wikilinks.append(wikilink) - pdf_wikilink = obsidian_wikilink_for_pdf(resolved_pdf, vault, zotero_dir) if resolved_pdf else '' - content = library_record_markdown({'zotero_key': item['key'], 'domain': domain, 'title': item.get('title', ''), 'year': item.get('year', ''), 'doi': item.get('doi', ''), 'date': item.get('date', ''), 'collection_path': ' | '.join(item.get('collections', [])), 'collections': collection_meta.get('collections', []), 'collection_tags': collection_meta.get('collection_tags', []), 'collection_group': collection_meta.get('collection_group', []), 'has_pdf': has_pdf, 'pdf_path': pdf_wikilink, 'bbt_path_raw': item.get('bbt_path_raw', ''), 'zotero_storage_key': item.get('zotero_storage_key', ''), 'attachment_count': item.get('attachment_count', 0), 'supplementary': supplementary_wikilinks, 'path_error': item.get('path_error', ''), 'recommend_analyze': bool(pdf_attachments), 'analyze': existing.get('analyze', False), 'do_ocr': existing.get('do_ocr', False), 'ocr_status': record_ocr_status, 'fulltext_md_path': fulltext_md_path, 'deep_reading_status': 'done' if note_text and has_deep_reading_content(note_text) else 'pending', 'analysis_note': existing.get('analysis_note', ''), 'first_author': first_author, 'journal': journal, 'impact_factor': impact_factor}) - if record_path.exists(): - existing_content = record_path.read_text(encoding='utf-8') - updated_content = _add_missing_frontmatter_fields(existing_content, {'first_author': first_author, 'journal': journal, 'impact_factor': impact_factor, 'bbt_path_raw': item.get('bbt_path_raw', ''), 'zotero_storage_key': item.get('zotero_storage_key', ''), 'attachment_count': str(item.get('attachment_count', 0)), 'path_error': item.get('path_error', '')}) - updated_content = update_frontmatter_field(updated_content, 'has_pdf', has_pdf) - updated_content = update_frontmatter_field(updated_content, 'pdf_path', pdf_wikilink) - updated_content = update_frontmatter_field(updated_content, 'ocr_status', record_ocr_status) - updated_content = update_frontmatter_field(updated_content, 'deep_reading_status', 'done' if note_text and has_deep_reading_content(note_text) else 'pending') - updated_content = update_frontmatter_field(updated_content, 'fulltext_md_path', fulltext_md_path or '') - if updated_content != existing_content: - record_path.write_text(updated_content, encoding='utf-8') - updated += 1 - else: - written += 1 - record_path.write_text(content, encoding='utf-8') - print(f'selection-sync: wrote {written} records, updated {updated} records') - return 0 - -def load_candidates_by_id(paths: dict[str, Path]) -> dict[str, dict]: - candidates = read_json(paths['candidates']) - return {row['candidate_id']: row for row in candidates} - -def save_candidates(paths: dict[str, Path], candidate_map: dict[str, dict]) -> None: - collection_catalog = load_domain_collection_catalog(paths) - export_inventory = load_export_inventory(paths) - rows = [] - for row in candidate_map.values(): - copy = dict(row) - copy['decision'] = canonicalize_decision(copy.get('decision', '')) - copy = apply_candidate_collection_resolution(copy, collection_catalog) - copy = apply_existing_library_match(copy, export_inventory) - copy['final_collection'] = compute_final_collection(copy) - rows.append(copy) - write_json(paths['candidates'], rows) - -def writeback_command_for_candidate(row: dict) -> dict | None: - final_collection = str(row.get('final_collection', '') or '').strip() - if not final_collection: - return None - candidate_id = str(row.get('candidate_id', '') or '').strip() - if not candidate_id: - return None - command = {'command_id': f'wb-native-{candidate_id}', 'status': 'queued', 'source_candidate_id': candidate_id, 'target_domain': str(row.get('domain', '') or '').strip(), 'target_collection': final_collection, 'requested_at': datetime.now(timezone.utc).isoformat()} - existing_zotero_key = str(row.get('existing_zotero_key', '') or '').strip() - if existing_zotero_key: - command.update({'action': 'attach_existing_item_to_collection', 'existing_zotero_key': existing_zotero_key}) - return command - doi = str(row.get('doi', '') or '').strip() - pmid = str(row.get('pmid', '') or '').strip() - if doi: - command.update({'action': 'create_item_from_identifier', 'identifier_type': 'doi', 'identifier': doi, 'metadata_fallback': {'title': row.get('title', ''), 'authors': row.get('authors', []), 'year': str(row.get('year', '') or ''), 'journal': row.get('journal', ''), 'doi': doi, 'pmid': pmid, 'abstractNote': row.get('abstract_short', '')}}) - return command - if pmid: - command.update({'action': 'create_item_from_identifier', 'identifier_type': 'pmid', 'identifier': pmid, 'metadata_fallback': {'title': row.get('title', ''), 'authors': row.get('authors', []), 'year': str(row.get('year', '') or ''), 'journal': row.get('journal', ''), 'doi': doi, 'pmid': pmid, 'abstractNote': row.get('abstract_short', '')}}) - return command - command.update({'action': 'create_item_from_metadata', 'metadata': {'title': row.get('title', ''), 'authors': row.get('authors', []), 'year': str(row.get('year', '') or ''), 'journal': row.get('journal', ''), 'doi': doi, 'pmid': pmid, 'abstractNote': row.get('abstract_short', '')}}) - return command - -def sync_writeback_queue(paths: dict[str, Path], candidate_map: dict[str, dict]) -> tuple[list[dict], int]: - existing_rows = read_jsonl(paths['queue']) - existing_by_candidate = {str(row.get('source_candidate_id', '') or '').strip(): dict(row) for row in existing_rows if str(row.get('source_candidate_id', '') or '').strip()} - queue_rows: list[dict] = [] - queued_candidates: set[str] = set() - created = 0 - for candidate_id, row in candidate_map.items(): - decision = canonicalize_decision(row.get('decision', '')) - if decision != '纳入': - continue - if str(row.get('import_status', '') or '').strip() == 'imported': - continue - candidate_copy = dict(row) - candidate_copy['final_collection'] = compute_final_collection(candidate_copy) - final_collection = str(candidate_copy.get('final_collection', '') or '').strip() - if not final_collection: - candidate_copy['import_status'] = 'needs_collection_resolution' - candidate_map[candidate_id] = candidate_copy - continue - command = writeback_command_for_candidate(candidate_copy) - if not command: - candidate_copy['import_status'] = 'blocked' - candidate_map[candidate_id] = candidate_copy - continue - existing = existing_by_candidate.get(candidate_id) - if existing and str(existing.get('status', '') or '').strip() in {'queued', 'running', 'processed'}: - merged = dict(existing) - merged['target_collection'] = command['target_collection'] - merged['target_domain'] = command.get('target_domain', merged.get('target_domain', '')) - if merged.get('status') != 'processed': - merged['requested_at'] = command['requested_at'] - queue_rows.append(merged) - else: - queue_rows.append(command) - created += 1 - candidate_copy['import_status'] = 'queued_for_writeback' - candidate_map[candidate_id] = candidate_copy - queued_candidates.add(candidate_id) - for row in existing_rows: - candidate_id = str(row.get('source_candidate_id', '') or '').strip() - status = str(row.get('status', '') or '').strip() - if candidate_id in queued_candidates: - continue - if status == 'processed': - queue_rows.append(row) - write_jsonl(paths['queue'], queue_rows) - return (queue_rows, created) - -def load_bridge_config(paths: dict[str, Path]) -> dict: - config_path = paths['bridge_config'] - if not config_path.exists(): - sample = read_json(paths['bridge_config_sample']) - write_json(config_path, sample) - return read_json(config_path) - -def apply_writeback_log(paths: dict[str, Path], candidate_map: dict[str, dict]) -> int: - log_rows = read_jsonl(paths['log']) - changed = 0 - latest_by_candidate: dict[str, dict] = {} - for row in log_rows: - candidate_id = str(row.get('source_candidate_id', '') or '').strip() - if candidate_id: - latest_by_candidate[candidate_id] = row - for candidate_id, log_row in latest_by_candidate.items(): - candidate = dict(candidate_map.get(candidate_id, {})) - if not candidate: - continue - status = str(log_row.get('status', '') or '').strip() - if status == 'success': - candidate['import_status'] = 'imported' - candidate['zotero_key'] = str(log_row.get('zotero_key', '') or '').strip() - changed += 1 - elif status == 'error': - candidate['import_status'] = 'writeback_error' - changed += 1 - candidate_map[candidate_id] = candidate - return changed - -def invoke_native_bridge(paths: dict[str, Path], max_commands: int=5) -> dict: - config = load_bridge_config(paths) - base_url = str(config.get('server_base_url', 'http://127.0.0.1:23119')).rstrip('/') - endpoint = str(config.get('process_endpoint', '/literaturePipeline/processQueue')) - payload = {'queuePath': str(paths['queue']).replace('\\', '/'), 'logPath': str(paths['log']).replace('\\', '/'), 'configPath': str(paths['bridge_config']).replace('\\', '/'), 'maxCommands': max_commands} - response = requests.post(f'{base_url}{endpoint}', json=payload, timeout=30) - response.raise_for_status() - result = response.json() - if not result.get('ok', False): - raise RuntimeError(result.get('error', 'Native bridge returned non-ok result')) - return result - -def normalize_candidate_title(text: str) -> str: - return re.sub('\\s+', ' ', str(text or '').strip().lower()) - -def candidate_identity_keys(row: dict) -> dict[str, str]: - doi = str(row.get('doi', '') or '').strip().lower() - pmid = str(row.get('pmid', '') or '').strip() - title = normalize_candidate_title(row.get('title', '')) - return {'doi': doi, 'pmid': pmid, 'title': title} - -def candidate_id_from_payload(row: dict) -> str: - source = str(row.get('source', '') or 'candidate').strip().lower() - doi = re.sub('[^a-z0-9]+', '-', str(row.get('doi', '') or '').strip().lower()).strip('-') - pmid = re.sub('[^0-9]+', '', str(row.get('pmid', '') or '').strip()) - fallback = re.sub('[^a-z0-9]+', '-', normalize_candidate_title(row.get('title', ''))).strip('-')[:80] - suffix = doi or pmid or fallback or datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') - return f'{source}-{suffix}' - -def _normalize_candidate_value(value): - if value is None: - return '' - if isinstance(value, list): - return [str(item).strip() for item in value if str(item).strip()] - return str(value).strip() - -def _authors_from_pubmed(value) -> list[str] | str: - if isinstance(value, list): - return [str(item).strip() for item in value if str(item).strip()] - return str(value or '').strip() - -def _authors_from_openalex(value) -> list[str]: - authors = [] - if isinstance(value, list): - for item in value: - if isinstance(item, dict): - author = item.get('author') or {} - name = author.get('display_name') or item.get('display_name') or '' - if name: - authors.append(str(name).strip()) - elif str(item).strip(): - authors.append(str(item).strip()) - return authors - -def _abstract_from_openalex(inverted_index) -> str: - if not isinstance(inverted_index, dict): - return '' - tokens: list[tuple[int, str]] = [] - for word, positions in inverted_index.items(): - if not isinstance(positions, list): - continue - for pos in positions: - try: - tokens.append((int(pos), str(word))) - except Exception: - continue - if not tokens: - return '' - tokens.sort(key=lambda item: item[0]) - return ' '.join((word for _, word in tokens)) - -def _authors_from_arxiv(value) -> list[str]: - authors = [] - if isinstance(value, list): - for item in value: - if isinstance(item, dict): - name = item.get('name', '') or item.get('author', '') - if name: - authors.append(str(name).strip()) - elif str(item).strip(): - authors.append(str(item).strip()) - return authors - -def _authors_from_scholar(value) -> list[str]: - if isinstance(value, list): - return [str(item).strip() for item in value if str(item).strip()] - if isinstance(value, str): - parts = re.split('\\s*,\\s*|\\s+and\\s+', value) - return [part.strip() for part in parts if part.strip()] - return [] - -def adapt_pubmed_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - adapted = {'candidate_id': row.get('candidate_id') or f"pubmed-{payload.get('pmid') or payload.get('PMID') or ''}", 'domain': row.get('domain', ''), 'title': payload.get('title', ''), 'authors': _authors_from_pubmed(payload.get('authors', [])), 'year': payload.get('year', ''), 'journal': payload.get('journal', ''), 'doi': payload.get('doi', ''), 'pmid': payload.get('pmid') or payload.get('PMID', ''), 'source': row.get('source', '') or 'pubmed_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': payload.get('abstract', '') or payload.get('abstract_short', ''), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} - return adapted - -def adapt_openalex_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - primary_location = payload.get('primary_location') or {} - source_info = primary_location.get('source') or {} - openalex_id = str(payload.get('id', '') or '').rstrip('/').split('/')[-1] - adapted = {'candidate_id': row.get('candidate_id') or f'openalex-{openalex_id}', 'domain': row.get('domain', ''), 'title': payload.get('display_name', '') or payload.get('title', ''), 'authors': _authors_from_openalex(payload.get('authorships', [])), 'year': payload.get('publication_year', '') or payload.get('year', ''), 'journal': source_info.get('display_name', '') or payload.get('journal', ''), 'doi': str(payload.get('doi', '') or '').replace('https://doi.org/', ''), 'pmid': '', 'source': row.get('source', '') or 'openalex_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': _abstract_from_openalex(payload.get('abstract_inverted_index')), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} - return adapted - -def adapt_arxiv_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - arxiv_id = str(payload.get('id', '') or payload.get('entry_id', '')).rstrip('/').split('/')[-1] - adapted = {'candidate_id': row.get('candidate_id') or f'arxiv-{arxiv_id}', 'domain': row.get('domain', ''), 'title': payload.get('title', ''), 'authors': _authors_from_arxiv(payload.get('authors', [])), 'year': str(payload.get('published', '') or '')[:4], 'journal': payload.get('journal_ref', '') or 'arXiv', 'doi': payload.get('doi', ''), 'pmid': '', 'source': row.get('source', '') or 'arxiv_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': payload.get('summary', '') or payload.get('abstract', ''), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} - return adapted - -def adapt_google_scholar_candidate(row: dict) -> dict: - payload = dict(row.get('payload') or {}) - adapted = {'candidate_id': row.get('candidate_id') or f"google-scholar-{re.sub('[^a-z0-9]+', '-', normalize_candidate_title(payload.get('title', ''))).strip('-')[:80]}", 'domain': row.get('domain', ''), 'title': payload.get('title', ''), 'authors': _authors_from_scholar(payload.get('authors', [])), 'year': str(payload.get('year', '') or ''), 'journal': payload.get('journal', '') or payload.get('venue', ''), 'doi': payload.get('doi', ''), 'pmid': '', 'source': row.get('source', '') or 'google_scholar_search', 'requester_skill': row.get('requester_skill', ''), 'request_context': row.get('request_context', ''), 'abstract_short': payload.get('abstract', '') or payload.get('snippet', ''), 'decision': row.get('decision', ''), 'recommended_collection': row.get('recommended_collection', ''), 'recommend_confidence': row.get('recommend_confidence', ''), 'recommend_reason': row.get('recommend_reason', ''), 'user_collection': row.get('user_collection', ''), 'final_collection': row.get('final_collection', ''), 'duplicate_hint': row.get('duplicate_hint', ''), 'import_status': row.get('import_status', ''), 'note': row.get('note', ''), 'candidate_source_type': row.get('candidate_source_type', '') or 'external_search', 'source_context': row.get('source_context', ''), 'task_relevance_reason': row.get('task_relevance_reason', '')} - return adapted - -def adapt_candidate_event(row: dict) -> dict: - adapter = str(row.get('adapter', '') or '').strip() - if adapter == 'pubmed_search': - return adapt_pubmed_candidate(row) - if adapter == 'openalex_search': - return adapt_openalex_candidate(row) - if adapter == 'arxiv_search': - return adapt_arxiv_candidate(row) - if adapter == 'google_scholar_search': - return adapt_google_scholar_candidate(row) - return dict(row) - -def default_user_agent() -> str: - return 'ResearchLiteraturePipeline/1.0 (+local-vault)' - -def build_search_event(base_task: dict, source: str, payload: dict) -> dict: - return {'adapter': source, 'payload': payload, 'source': source, 'requester_skill': base_task.get('requester_skill', ''), 'request_context': base_task.get('request_context', ''), 'domain': base_task.get('domain', ''), 'recommended_collection': base_task.get('recommended_collection', ''), 'recommend_confidence': base_task.get('recommend_confidence', ''), 'recommend_reason': base_task.get('recommend_reason', '') or f'{source} 检索命中', 'candidate_source_type': base_task.get('candidate_source_type', '') or 'external_search', 'task_relevance_reason': base_task.get('task_relevance_reason', ''), 'note': base_task.get('note', '')} - -def _pubmed_abstract_and_doi_map(xml_text: str) -> dict[str, dict]: - root = ET.fromstring(xml_text) - result = {} - for article in root.findall('.//PubmedArticle'): - pmid = (article.findtext('.//MedlineCitation/PMID') or '').strip() - if not pmid: - continue - abstract_parts = [] - for node in article.findall('.//Abstract/AbstractText'): - label = (node.attrib.get('Label') or '').strip() - text = ' '.join(''.join(node.itertext()).split()) - if not text: - continue - abstract_parts.append(f'{label}: {text}' if label else text) - doi = '' - for id_node in article.findall('.//PubmedData/ArticleIdList/ArticleId'): - if (id_node.attrib.get('IdType') or '').lower() == 'doi': - doi = ''.join(id_node.itertext()).strip() - if doi: - break - if not doi: - for id_node in article.findall('.//ELocationID'): - if (id_node.attrib.get('EIdType') or '').lower() == 'doi': - doi = ''.join(id_node.itertext()).strip() - if doi: - break - result[pmid] = {'abstract': '\n'.join(abstract_parts).strip(), 'doi': doi} - return result - -def search_pubmed(task: dict, limit: int) -> tuple[list[dict], dict]: - query = str(task.get('query', '') or '').strip() - if not query: - return ([], {'count': 0, 'ids': []}) - base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils' - headers = {'User-Agent': os.environ.get('LIT_PIPELINE_USER_AGENT', default_user_agent())} - common_params = {} - email = os.environ.get('NCBI_EMAIL', '').strip() - api_key = os.environ.get('NCBI_API_KEY', '').strip() - if email: - common_params['email'] = email - if api_key: - common_params['api_key'] = api_key - esearch = requests.get(f'{base_url}/esearch.fcgi', params={'db': 'pubmed', 'retmode': 'json', 'sort': 'relevance', 'term': query, 'retmax': limit, **common_params}, headers=headers, timeout=60) - esearch.raise_for_status() - search_payload = esearch.json().get('esearchresult', {}) - ids = [str(item).strip() for item in search_payload.get('idlist', []) if str(item).strip()] - if not ids: - return ([], {'count': 0, 'ids': []}) - id_text = ','.join(ids) - esummary = requests.get(f'{base_url}/esummary.fcgi', params={'db': 'pubmed', 'retmode': 'json', 'id': id_text, **common_params}, headers=headers, timeout=60) - esummary.raise_for_status() - summary_payload = esummary.json().get('result', {}) - efetch = requests.get(f'{base_url}/efetch.fcgi', params={'db': 'pubmed', 'retmode': 'xml', 'id': id_text, **common_params}, headers=headers, timeout=90) - efetch.raise_for_status() - abstract_map = _pubmed_abstract_and_doi_map(efetch.text) - rows = [] - for pmid in ids: - item = summary_payload.get(pmid, {}) or {} - title = html.unescape(str(item.get('title', '') or '')).strip() - if not title: - continue - article_ids = item.get('articleids', []) or [] - doi = '' - for article_id in article_ids: - if str(article_id.get('idtype', '')).lower() == 'doi': - doi = str(article_id.get('value', '')).strip() - if doi: - break - if not doi: - doi = abstract_map.get(pmid, {}).get('doi', '') - rows.append({'pmid': pmid, 'title': title, 'authors': [author.get('name', '') for author in item.get('authors') or [] if author.get('name')], 'year': _extract_year(str(item.get('pubdate', '') or '')), 'journal': item.get('fulljournalname', '') or item.get('source', ''), 'doi': doi, 'abstract': abstract_map.get(pmid, {}).get('abstract', '')}) - return (rows, {'count': len(rows), 'ids': ids}) - -def search_openalex(task: dict, limit: int) -> tuple[list[dict], dict]: - query = str(task.get('query', '') or '').strip() - if not query: - return ([], {'count': 0}) - headers = {'User-Agent': os.environ.get('LIT_PIPELINE_USER_AGENT', default_user_agent())} - params = {'search': query, 'per-page': limit} - api_key = os.environ.get('OPENALEX_API_KEY', '').strip() - if api_key: - params['api_key'] = api_key - mailto = os.environ.get('OPENALEX_MAILTO', '').strip() - if mailto: - params['mailto'] = mailto - response = requests.get('https://api.openalex.org/works', params=params, headers=headers, timeout=60) - response.raise_for_status() - payload = response.json() - results = payload.get('results', []) or [] - return (results, {'count': len(results), 'meta': payload.get('meta', {})}) - -def search_arxiv(task: dict, limit: int) -> tuple[list[dict], dict]: - query = str(task.get('query', '') or '').strip() - if not query: - return ([], {'count': 0}) - encoded_query = urllib.parse.quote(f'all:{query}') - url = f'https://export.arxiv.org/api/query?search_query={encoded_query}&start=0&max_results={limit}&sortBy=relevance&sortOrder=descending' - headers = {'User-Agent': os.environ.get('LIT_PIPELINE_USER_AGENT', default_user_agent())} - response = requests.get(url, headers=headers, timeout=60) - response.raise_for_status() - ns = {'atom': 'http://www.w3.org/2005/Atom', 'arxiv': 'http://arxiv.org/schemas/atom'} - root = ET.fromstring(response.text) - entries = [] - for entry in root.findall('atom:entry', ns): - entries.append({'id': (entry.findtext('atom:id', default='', namespaces=ns) or '').strip(), 'title': ' '.join((entry.findtext('atom:title', default='', namespaces=ns) or '').split()), 'summary': ' '.join((entry.findtext('atom:summary', default='', namespaces=ns) or '').split()), 'published': (entry.findtext('atom:published', default='', namespaces=ns) or '').strip(), 'authors': [{'name': (node.findtext('atom:name', default='', namespaces=ns) or '').strip()} for node in entry.findall('atom:author', ns)], 'doi': (entry.findtext('arxiv:doi', default='', namespaces=ns) or '').strip(), 'journal_ref': (entry.findtext('arxiv:journal_ref', default='', namespaces=ns) or '').strip()}) - return (entries, {'count': len(entries)}) - -def _coerce_source_name(value: str) -> str: - text = str(value or '').strip().lower() - aliases = {'pubmed': 'pubmed_search', 'pubmed_search': 'pubmed_search', 'openalex': 'openalex_search', 'openalex_search': 'openalex_search', 'arxiv': 'arxiv_search', 'arxiv_search': 'arxiv_search'} - return aliases.get(text, text) - -def run_search_command(vault: Path, args) -> int: - paths = pipeline_paths(vault) - paths['search_tasks'].mkdir(parents=True, exist_ok=True) - task_id = f"search-{datetime.now().strftime('%Y%m%d-%H%M%S')}" - sources = args.sources or ['pubmed_search', 'openalex_search', 'arxiv_search'] - task = {'task_id': task_id, 'query': args.query, 'domain': args.domain, 'recommended_collection': args.recommended_collection or '', 'requester_skill': args.requester_skill or '', 'request_context': args.request_context or '', 'sources': sources, 'limit': args.limit, 'recommend_reason': args.recommend_reason or '围绕检索主题补充候选文献', 'candidate_source_type': 'external_search'} - task_path = paths['search_tasks'] / f'{task_id}.json' - write_json(task_path, task) - print(f'search: task written -> {task_path}') - code = run_search_sources(vault) - if code: - return code - if not args.skip_ingest: - return run_ingest_candidates(vault) - return 0 - -def normalize_candidate_payload(row: dict) -> dict: - normalized = {key: _normalize_candidate_value(value) for key, value in adapt_candidate_event(row).items()} - normalized['candidate_id'] = str(normalized.get('candidate_id', '') or '').strip() or candidate_id_from_payload(normalized) - normalized['title'] = str(normalized.get('title', '') or '').strip() - normalized['domain'] = str(normalized.get('domain', '') or '').strip() - normalized['source'] = str(normalized.get('source', '') or '').strip() or 'candidate_ingest' - normalized['candidate_source_type'] = str(normalized.get('candidate_source_type', '') or '').strip() or normalized['source'] - normalized['decision'] = canonicalize_decision(normalized.get('decision', '')) - normalized['import_status'] = str(normalized.get('import_status', '') or '').strip() or 'pending' - normalized['recommend_confidence'] = str(normalized.get('recommend_confidence', '') or '').strip() or '0' - normalized['status'] = 'candidate' - return normalized - -def merge_candidate_record(existing: dict | None, incoming: dict) -> dict: - merged = dict(existing or {}) - preserve_if_existing = {'decision', 'user_collection', 'note', 'import_status'} - for key, value in incoming.items(): - if existing and key in preserve_if_existing: - current = merged.get(key, '') - if str(current).strip(): - continue - merged[key] = value - merged['decision'] = canonicalize_decision(merged.get('decision', '')) - merged['final_collection'] = compute_final_collection(merged) - return merged - -def resolve_existing_candidate(candidate_map: dict[str, dict], incoming: dict) -> tuple[str | None, dict | None]: - candidate_id = incoming.get('candidate_id', '') - if candidate_id and candidate_id in candidate_map: - return (candidate_id, candidate_map[candidate_id]) - incoming_keys = candidate_identity_keys(incoming) - for existing_id, existing in candidate_map.items(): - existing_keys = candidate_identity_keys(existing) - if incoming_keys['doi'] and incoming_keys['doi'] == existing_keys['doi']: - return (existing_id, existing) - if incoming_keys['pmid'] and incoming_keys['pmid'] == existing_keys['pmid']: - return (existing_id, existing) - if incoming_keys['title'] and incoming_keys['title'] == existing_keys['title']: - return (existing_id, existing) - return (None, None) - -def _harvest_csv_paths(paths: dict[str, Path]) -> list[Path]: - root = paths['harvest_root'] - if not root.exists(): - return [] - return sorted(root.rglob('*-05-reference-harvest-candidates.csv')) - -def _normalize_harvest_value(value: str) -> str: - text = str(value or '').strip() - if text == '': - return '' - return text - -def _normalize_harvest_row(row: dict[str, str]) -> dict: - normalized = normalize_candidate_payload({key: _normalize_harvest_value(value) for key, value in row.items()}) - normalized['source'] = normalized.get('source', '') or 'reference_harvest' - normalized['candidate_source_type'] = normalized.get('candidate_source_type', '') or 'reference_harvest' - return normalized - -def _merge_harvest_candidate(existing: dict | None, incoming: dict) -> dict: - return merge_candidate_record(existing, incoming) - -def next_key(domain: str, export_rows: list[dict]) -> str: - prefix = 'ORTHO' if domain == '骨科' else 'SPORT' - existing = [row.get('key', '') for row in export_rows] - max_num = 0 - for key in existing: - if key.startswith(prefix): - suffix = key[len(prefix):] - if suffix.isdigit(): - max_num = max(max_num, int(suffix)) - return f'{prefix}{max_num + 1:03d}' - -def frontmatter_note(entry: dict, existing_text: str='') -> str: - deep_reading_path = entry.get('deep_reading_md_path', '') - preserved_deep = extract_preserved_deep_reading(existing_text) - lines = ['---', f"title: {yaml_quote(entry['title'])}", f"year: {entry.get('year', '')}", 'type: article', f"journal: {yaml_quote(entry.get('journal', ''))}", 'authors:'] - for author in entry.get('authors', []): - lines.append(f' - {yaml_quote(author)}') - lines.extend([f"collection_path: {yaml_quote(entry.get('collection_path', ''))}", f"domain: {yaml_quote(entry.get('domain', ''))}", f"zotero_key: {yaml_quote(entry.get('zotero_key', ''))}", f"doi: {yaml_quote(entry.get('doi', ''))}", f"pmid: {yaml_quote(entry.get('pmid', ''))}"]) - lines.extend(yaml_list('collection_group', entry.get('collection_group', []))) - lines.extend(yaml_list('collections', entry.get('collections', []))) - lines.extend(yaml_list('collection_tags', entry.get('collection_tags', []))) - lines.extend(yaml_block(entry.get('abstract', ''))) - lines.extend([f"has_pdf: {('true' if entry.get('has_pdf') else 'false')}", f"ocr_status: {yaml_quote(entry.get('ocr_status', 'pending'))}", f"ocr_job_id: {yaml_quote(entry.get('ocr_job_id', ''))}", f"ocr_md_path: {yaml_quote(entry.get('ocr_md_path', ''))}", f"ocr_json_path: {yaml_quote(entry.get('ocr_json_path', ''))}", f"deep_reading_status: {yaml_quote(entry.get('deep_reading_status', 'pending'))}", f'deep_reading_md_path: {yaml_quote(deep_reading_path)}', f"pdf_path: {yaml_quote(entry.get('pdf_path', ''))}", 'tags:', ' - 文献阅读', f" - {entry.get('domain', '')}", '---', '', f"# {entry['title']}", '', '## 📄 文献基本信息', '', f"- Zotero Key: `{entry.get('zotero_key', '')}`", f"- Collection: `{entry.get('collection_path', '')}`", f"- 作者：{', '.join(entry.get('authors', []))}", f"- PDF: {('已检测' if entry.get('has_pdf') else '未检测到')}", f"- OCR: {entry.get('ocr_status', 'pending')}", f"- 精读: {entry.get('deep_reading_status', 'pending')}", '', '## 摘要', '', entry.get('abstract', '') or '暂无摘要', '', '## 💡 文献内容总结', '', '- 由 sync worker 自动生成的正式文献卡片。', '- 精读笔记（Deep Reading）仅由 /pf-deep 命令维护；sync --index 只保留已有内容，不自动生成。', '- 如需精读，请在 Base 中勾选 analyze，OCR 完成后运行 /pf-deep 。', '']) - if preserved_deep: - lines.extend(['', preserved_deep, '']) - return '\n'.join(lines) - -def analyze_selected_keys(paths: dict[str, Path]) -> set[str]: - return {key for key, row in load_control_actions(paths).items() if row.get('analyze')} - -def run_index_refresh(vault: Path) -> int: - paths = pipeline_paths(vault) - config = load_domain_config(paths) - ensure_base_views(vault, paths, config) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} - from paperforge.config import load_vault_config as _load_vault_config - cfg = _load_vault_config(vault) - zotero_dir = vault / cfg.get('system_dir', '99_System') / 'Zotero' - exports = {} - for export_path in sorted(paths['exports'].glob('*.json')): - domain = domain_lookup.get(export_path.name, export_path.stem) - export_rows = load_export_rows(export_path) - exports[domain] = {row['key']: row for row in export_rows} - selected_keys = None - index_rows = [] - lit_root = paths['literature'] - for export_path in sorted(paths['exports'].glob('*.json')): - domain = domain_lookup.get(export_path.name, export_path.stem) - export_rows = load_export_rows(export_path) - for item in export_rows: - key = item['key'] - if selected_keys is not None and key not in selected_keys: - continue - collection_meta = collection_fields(item.get('collections', [])) - pdf_attachments = [a for a in item.get('attachments', []) if a.get('contentType') == 'application/pdf'] - meta_path = paths['ocr'] / key / 'meta.json' - meta = read_json(meta_path) if meta_path.exists() else {} - if meta: - validated_ocr_status, validated_error = validate_ocr_meta(paths, meta) - meta['ocr_status'] = validated_ocr_status - if validated_error: - meta['error'] = validated_error - write_json(meta_path, meta) - title_slug = slugify_filename(item['title']) - note_path = lit_root / domain / f'{key} - {title_slug}.md' - if note_path.parent.exists(): - for stale_note in note_path.parent.glob(f'{key} - *.md'): - if stale_note != note_path: - stale_note.unlink() - entry = {'zotero_key': key, 'domain': domain, 'title': item['title'], 'authors': item.get('authors', []), 'abstract': item.get('abstract', ''), 'journal': item.get('journal', ''), 'year': item.get('year', ''), 'doi': item.get('doi', ''), 'pmid': item.get('pmid', ''), 'collection_path': ' | '.join(item.get('collections', [])), 'collections': collection_meta.get('collections', []), 'collection_tags': collection_meta.get('collection_tags', []), 'collection_group': collection_meta.get('collection_group', []), 'has_pdf': bool(pdf_attachments), 'pdf_path': obsidian_wikilink_for_pdf(pdf_attachments[0]['path'], vault, zotero_dir) if pdf_attachments else '', 'ocr_status': meta.get('ocr_status', 'pending'), 'ocr_job_id': meta.get('ocr_job_id', ''), 'ocr_md_path': obsidian_wikilink_for_path(vault, meta.get('markdown_path', '')), 'ocr_json_path': meta.get('json_path', ''), 'deep_reading_status': 'done' if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding='utf-8')) else 'pending', 'note_path': str(note_path.relative_to(vault)).replace('\\', '/'), 'deep_reading_md_path': str(note_path.relative_to(vault)).replace('\\', '/') if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding='utf-8')) else ''} - note_path.parent.mkdir(parents=True, exist_ok=True) - existing_text = note_path.read_text(encoding='utf-8') if note_path.exists() else '' - note_path.write_text(frontmatter_note(entry, existing_text), encoding='utf-8') - index_rows.append(entry) - write_json(paths['index'], index_rows) - print(f'index-refresh: wrote {len(index_rows)} index rows') - control_records_dir = paths['library_records'] - if control_records_dir.exists(): - for domain_dir in control_records_dir.iterdir(): - if not domain_dir.is_dir(): - continue - domain = domain_dir.name - domain_export_keys = set(exports.get(domain, {}).keys()) - records_by_title = {} - records_info = {} - for record_file in domain_dir.glob('*.md'): - try: - content = record_file.read_text(encoding='utf-8') - title_match = re.search('^title:\\s*["\\\']?(.+)["\\\']?\\s*$', content, re.MULTILINE) - title = title_match.group(1) if title_match else '' - has_pdf = 'has_pdf: true' in content - normalized = re.sub('[^a-z0-9]', '', title.lower())[:20] - key = record_file.stem - records_info[key] = {'file': record_file, 'title': title, 'has_pdf': has_pdf, 'normalized': normalized} - if normalized not in records_by_title: - records_by_title[normalized] = [] - records_by_title[normalized].append(key) - except Exception: - continue - to_delete = [] - for normalized, keys in records_by_title.items(): - keys_in_export = [k for k in keys if k in domain_export_keys] - keys_not_in_export = [k for k in keys if k not in domain_export_keys] - if keys_in_export and keys_not_in_export: - for k in keys_not_in_export: - if not records_info[k]['has_pdf']: - to_delete.append(k) - deleted_count = 0 - for key in to_delete: - try: - records_info[key]['file'].unlink() - deleted_count += 1 - except Exception: - pass - if deleted_count > 0: - print(f'index-refresh: cleaned {deleted_count} orphaned records in {domain}') - return 0 - -def ensure_ocr_meta(vault: Path, row: dict) -> dict: - paths = pipeline_paths(vault) - key = row['zotero_key'] - meta_path = paths['ocr'] / key / 'meta.json' - meta_path.parent.mkdir(parents=True, exist_ok=True) - meta = read_json(meta_path) if meta_path.exists() else {} - meta.setdefault('zotero_key', key) - meta.setdefault('source_pdf', row.get('pdf_path', '')) - meta.setdefault('ocr_provider', 'PaddleOCR-VL-1.5') - meta.setdefault('mode', 'async') - meta.setdefault('ocr_status', 'pending') - meta.setdefault('ocr_job_id', '') - meta.setdefault('ocr_started_at', '') - meta.setdefault('ocr_finished_at', '') - meta.setdefault('page_count', 0) - meta.setdefault('markdown_path', '') - meta.setdefault('json_path', '') - try: - assets_path = str((paths['ocr'] / key / 'images').relative_to(vault)).replace('\\', '/') - except ValueError: - assets_path = str(paths['ocr'] / key / 'images') - meta.setdefault('assets_path', assets_path) - meta.setdefault('fulltext_md_path', '') - meta.setdefault('error', '') - return meta - -def validate_ocr_meta(paths: dict[str, Path], meta: dict) -> tuple[str, str]: - status = str(meta.get('ocr_status', 'pending') or 'pending').strip().lower() - if status != 'done': - return (status, str(meta.get('error', '') or '')) - key = str(meta.get('zotero_key', '') or '').strip() - if not key: - return ('done_incomplete', 'Missing zotero_key in OCR meta') - ocr_root = paths['ocr'] / key - fulltext_path = ocr_root / 'fulltext.md' - json_path = ocr_root / 'json' / 'result.json' - page_count = int(meta.get('page_count', 0) or 0) - if not fulltext_path.exists(): - return ('done_incomplete', 'OCR fulltext.md missing') - if not json_path.exists(): - return ('done_incomplete', 'OCR result.json missing') - fulltext_size = fulltext_path.stat().st_size - json_size = json_path.stat().st_size - if page_count < 1: - return ('done_incomplete', 'OCR page_count invalid') - if fulltext_size < 500: - return ('done_incomplete', 'OCR fulltext.md too small') - if json_size < 1000: - return ('done_incomplete', 'OCR result.json too small') - try: - rendered_pages = fulltext_path.read_text(encoding='utf-8').count(''] - reference_blocks: list[dict] = [] - reference_continuations: list[dict] = [] - footnotes: list[str] = [] - footnote_counter = 0 - rendered_cluster_ids: set[int] = set() - rendered_caption_media_ids: set[int] = set() - first_page_meta_done = page_index != 1 - affiliation_buffer: list[str] = [] - deferred_meta: list[str] = [] - for block in blocks: - label = block.get('block_label', '') - content = block.get('block_content', '') - bbox = block.get('block_bbox', [0, 0, 0, 0]) - composite_region = composite_by_block_id.get(block.get('block_id')) - if composite_region: - if not composite_region.get('rendered'): - composite_region['rendered'] = True - region_bbox = composite_region['bbox'] - asset_path = images_dir / 'blocks' / f'page_{page_index:03d}_figure_{region_bbox[0]}_{region_bbox[1]}_{region_bbox[2]}_{region_bbox[3]}.jpg' - if page_image and crop_block_asset(page_image, region_bbox, asset_path): - rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault))) - continue - if label in {'header', 'header_image', 'footer', 'footer_image', 'number'}: - continue - if label == 'doc_title': - rendered.append(f'# {clean_block_text(content)}') - continue - if label == 'paragraph_title': - title = clean_block_text(content) - if title.lower() == 'check for updates' or is_subfigure_label(title) or is_reference_tail_noise_line(title) or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height): - continue - if page_index == 1 and affiliation_buffer: - rendered.extend(affiliation_buffer) - affiliation_buffer.clear() - if page_index == 1 and deferred_meta: - rendered.extend(deferred_meta) - deferred_meta.clear() - first_page_meta_done = True - rendered.append(f'### {title}') - continue - if label == 'abstract': - if page_index == 1 and affiliation_buffer: - rendered.extend(affiliation_buffer) - affiliation_buffer.clear() - rendered.append(clean_block_text(content)) - if page_index == 1 and deferred_meta: - rendered.extend(deferred_meta) - deferred_meta.clear() - first_page_meta_done = True - continue - if label == 'reference_content': - text = clean_block_text(content) - if text and (not is_reference_tail_noise_line(text)): - if parse_reference_number(text) is None: - reference_continuations.append(block) - else: - reference_blocks.append(block) - continue - if label == 'text': - text = clean_block_text(content) - if not text or is_subfigure_label(text) or is_frontmatter_noise_line(text) or is_reference_tail_noise_line(text) or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height): - continue - if raw_reference_blocks and bbox[1] >= first_reference_y - 10: - reference_continuations.append(block) - continue - if page_index == 1 and (not first_page_meta_done): - if handle_first_page_metadata_lines(text, rendered, affiliation_buffer, deferred_meta, footnotes): - continue - rendered.append(text) - continue - if label == 'display_formula': - formula = clean_block_text(content) - formula = formula.strip() - if formula.startswith('$$') and formula.endswith('$$') and (len(formula) >= 4): - formula = formula[2:-2].strip() - rendered.append(f'$$\n{formula}\n$$') - continue - if label == 'formula_number': - rendered.append(clean_block_text(content)) - continue - if label in {'table', 'image', 'chart'}: - if block.get('block_id') in caption_linked_media_ids: - continue - if block.get('block_id') in rendered_caption_media_ids: - continue - if label in {'image', 'chart'}: - cluster_id = cluster_index.get(block.get('block_id', -1)) - if cluster_id is None or cluster_id in rendered_cluster_ids: - continue - rendered_cluster_ids.add(cluster_id) - cluster_blocks = clusters[cluster_id] - bbox = [min((item['block_bbox'][0] for item in cluster_blocks)), min((item['block_bbox'][1] for item in cluster_blocks)), max((item['block_bbox'][2] for item in cluster_blocks)), max((item['block_bbox'][3] for item in cluster_blocks))] - asset_name = f'page_{page_index:03d}_figure_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg' - else: - asset_name = f'page_{page_index:03d}_{label}_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg' - asset_path = images_dir / 'blocks' / asset_name - if page_image and crop_block_asset(page_image, bbox, asset_path): - rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault))) - if label == 'table' and content: - rendered.append(clean_block_text(content)) - continue - if label == 'figure_title': - caption_text = clean_block_text(content) - if is_subfigure_label(caption_text): - continue - if not is_formal_figure_legend(caption_text): - continue - linked_media = figure_caption_map.get(block.get('block_id'), []) or table_caption_map.get(block.get('block_id'), []) - if linked_media and page_image: - rendered_caption_media_ids.update((item.get('block_id') for item in linked_media)) - union_bbox = [min((item['block_bbox'][0] for item in linked_media)), min((item['block_bbox'][1] for item in linked_media)), max((item['block_bbox'][2] for item in linked_media)), max((item['block_bbox'][3] for item in linked_media))] - asset_kind = 'figure' if block.get('block_id') in figure_caption_map else 'table' - asset_path = images_dir / 'blocks' / f'page_{page_index:03d}_{asset_kind}_{union_bbox[0]}_{union_bbox[1]}_{union_bbox[2]}_{union_bbox[3]}.jpg' - if crop_block_asset(page_image, union_bbox, asset_path): - rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault))) - if asset_kind == 'table': - for item in linked_media: - if item.get('block_label') == 'table' and item.get('block_content'): - rendered.append(clean_block_text(item.get('block_content', ''))) - break - rendered.append(caption_text) - continue - if label == 'vision_footnote': - if is_formal_figure_legend(content): - rendered.append(clean_block_text(content)) - continue - if is_embedded_vision_footnote_block(block, blocks, page_width=ocr_width, page_height=ocr_height): - continue - entries = parse_vision_footnote_entries(content) - marker_to_id = {} - for marker, body in entries: - footnote_counter += 1 - footnote_id = f'p{page_index}-fn{footnote_counter}' - marker_to_id[marker] = footnote_id - footnotes.append(f'[^{footnote_id}]: {body or clean_block_text(content)}') - if rendered and marker_to_id: - last = rendered[-1] - if last.startswith(''): - rendered[-1] = attach_table_footnotes(last, marker_to_id) - else: - for marker, footnote_id in marker_to_id.items(): - rendered[-1] = attach_footnote_reference(rendered[-1], marker, footnote_id) - continue - if footnotes: - rendered.append('') - rendered.extend(footnotes) - rendered = dedupe_page_media_lines(rendered) - if reference_blocks: - rendered.append('') - ordered_reference_blocks = sort_reference_blocks(reference_blocks) - continuation_map: dict[int, list[str]] = {} - sorted_continuations = sorted(reference_continuations, key=block_sort_key) - assigned_continuation_indexes: set[int] = set() - incomplete_reference_indexes = [index for index, block in enumerate(ordered_reference_blocks) if clean_block_text(block.get('block_content', '')).rstrip().endswith(':')] - for index, continuation in zip(incomplete_reference_indexes, sorted_continuations): - continuation_text = clean_block_text(continuation.get('block_content', '')) - if continuation_text: - continuation_map.setdefault(index, []).append(continuation_text) - assigned_continuation_indexes.add(id(continuation)) - for continuation in sorted_continuations: - if id(continuation) in assigned_continuation_indexes: - continue - target_index = assign_reference_continuation(continuation, ordered_reference_blocks) - if target_index is None: - continue - continuation_map.setdefault(target_index, []).append(clean_block_text(continuation.get('block_content', ''))) - assigned_continuation_indexes.add(id(continuation)) - unassigned_continuations = [clean_block_text(continuation.get('block_content', '')) for continuation in sorted_continuations if id(continuation) not in assigned_continuation_indexes] - for index, block in enumerate(ordered_reference_blocks): - text = clean_block_text(block.get('block_content', '')) - if text: - rendered.append(text) - for continuation_text in continuation_map.get(index, []): - if continuation_text: - rendered.append(continuation_text) - if text.rstrip().endswith(':') and unassigned_continuations: - rendered.append(unassigned_continuations.pop(0)) - return [part for part in rendered if part] - -def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tuple[int, str, str, str]: - paths = pipeline_paths(vault) - ocr_root = paths['ocr'] / key - json_dir = ocr_root / 'json' - images_dir = ocr_root / 'images' - page_cache_dir = ocr_root / 'pages' - meta_path = ocr_root / 'meta.json' - json_dir.mkdir(parents=True, exist_ok=True) - images_dir.mkdir(parents=True, exist_ok=True) - page_cache_dir.mkdir(parents=True, exist_ok=True) - page_num = 0 - merged_parts = [] - meta = read_json(meta_path) if meta_path.exists() else {} - source_pdf = Path(meta.get('source_pdf', '')) if meta.get('source_pdf') else None - pdf_doc = None - try: - if source_pdf and source_pdf.exists(): - pdf_doc = fitz.open(str(source_pdf)) - for page_payload in all_results: - for res in page_payload.get('layoutParsingResults', []): - page_num += 1 - merged_parts.append('\n\n'.join(render_page_blocks(vault, page_num, res, images_dir, page_cache_dir, pdf_doc=pdf_doc))) - finally: - if pdf_doc is not None: - pdf_doc.close() - write_json(json_dir / 'result.json', all_results) - fulltext_path = ocr_root / 'fulltext.md' - fulltext_path.write_text('\n\n'.join(merged_parts).strip() + '\n', encoding='utf-8') - markdown_dir = ocr_root / 'markdown' - if markdown_dir.exists(): - shutil.rmtree(markdown_dir) - markdown_path = str(fulltext_path.relative_to(vault)).replace('\\', '/') if page_num else '' - json_path = str((json_dir / 'result.json').relative_to(vault)).replace('\\', '/') - fulltext_md_path = str(fulltext_path.resolve()) - return (page_num, markdown_path, json_path, fulltext_md_path) - -def run_ocr(vault: Path) -> int: - from paperforge.pdf_resolver import resolve_pdf_path - paths = pipeline_paths(vault) - cleanup_blocked_ocr_dirs(paths) - control_actions = load_control_actions(paths) - target_keys = {key for key, action in control_actions.items() if action.get('do_ocr', False)} - target_rows = [] - for export_path in sorted(paths['exports'].glob('*.json')): - for item in load_export_rows(export_path): - if item['key'] not in target_keys: - continue - pdf_attachments = [a for a in item.get('attachments', []) if a.get('contentType') == 'application/pdf'] - target_rows.append({'zotero_key': item['key'], 'has_pdf': bool(pdf_attachments), 'pdf_path': pdf_attachments[0]['path'] if pdf_attachments else ''}) - ocr_queue = sync_ocr_queue(paths, target_rows) - max_items_raw = os.environ.get('PADDLEOCR_MAX_ITEMS', '').strip() - max_items = 3 - if max_items_raw: - try: - max_items = max(1, int(max_items_raw)) - except ValueError: - max_items = 3 - token = os.environ.get('PADDLEOCR_API_TOKEN', '').strip() - if not token: - token = os.environ.get('PADDLEOCR_API_TOKEN_USER', '').strip() - if not token: - try: - import winreg - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, 'Environment') as env_key: - token = str(winreg.QueryValueEx(env_key, 'PADDLEOCR_API_TOKEN')[0]).strip() - except Exception: - token = '' - job_url = os.environ.get('PADDLEOCR_JOB_URL', 'https://paddleocr.aistudio-app.com/api/v2/ocr/jobs').strip() - model = os.environ.get('PADDLEOCR_MODEL', 'PaddleOCR-VL-1.5').strip() - optional_payload = {'useDocOrientationClassify': False, 'useDocUnwarping': False, 'useChartRecognition': False} - changed = 0 - active_submitted = 0 - queue_changed = False - for queue_row in ocr_queue: - key = queue_row['zotero_key'] - meta = ensure_ocr_meta(vault, queue_row) - status = str(meta.get('ocr_status', 'pending') or 'pending').strip().lower() - queue_row['queue_status'] = status - if status == 'done': - queue_changed = True - continue - if status in {'queued', 'running'} and meta.get('ocr_job_id'): - active_submitted += 1 - if not token: - continue - response = requests.get(f"{job_url}/{meta['ocr_job_id']}", headers={'Authorization': f'bearer {token}'}, timeout=60) - response.raise_for_status() - try: - payload = response.json()['data'] - state = payload['state'] - except (json.JSONDecodeError, KeyError) as e: - from paperforge.ocr_diagnostics import classify_error - state, suggestion = classify_error(e, None) - meta['ocr_status'] = state - meta['error'] = f'API schema mismatch during polling: {e}' - meta['suggestion'] = suggestion - meta['raw_response'] = response.text[:1000] - queue_row['queue_status'] = state - write_json(paths['ocr'] / key / 'meta.json', meta) - changed += 1 - active_submitted = max(0, active_submitted - 1) - continue - if state in {'pending', 'running'}: - meta['ocr_status'] = state - queue_row['queue_status'] = state - meta['error'] = '' - elif state == 'done': - result_url = payload['resultUrl']['jsonUrl'] - result_response = requests.get(result_url, timeout=120) - result_response.raise_for_status() - lines = [line.strip() for line in result_response.text.splitlines() if line.strip()] - all_results = [] - for line in lines: - page_payload = json.loads(line)['result'] - all_results.append(page_payload) - page_num, markdown_path, json_path, fulltext_md_path = postprocess_ocr_result(vault, key, all_results) - meta['ocr_status'] = 'done' - meta['ocr_finished_at'] = datetime.now(timezone.utc).isoformat() - meta['page_count'] = page_num - meta['markdown_path'] = markdown_path - meta['json_path'] = json_path - meta['fulltext_md_path'] = fulltext_md_path - meta['error'] = '' - queue_row['queue_status'] = 'done' - queue_changed = True - active_submitted = max(0, active_submitted - 1) - else: - meta['ocr_status'] = 'error' - meta['error'] = payload.get('errorMsg', 'Unknown OCR failure') - queue_row['queue_status'] = 'error' - active_submitted = max(0, active_submitted - 1) - write_json(paths['ocr'] / key / 'meta.json', meta) - changed += 1 - available_slots = max(0, max_items - active_submitted) - if available_slots > 0: - for queue_row in ocr_queue: - if available_slots <= 0: - break - key = queue_row['zotero_key'] - meta = ensure_ocr_meta(vault, queue_row) - status = str(meta.get('ocr_status', 'pending') or 'pending').strip().lower() - if status == 'done': - queue_changed = True - continue - if status in {'queued', 'running'} and meta.get('ocr_job_id'): - continue - resolved_pdf = resolve_pdf_path( - queue_row.get('pdf_path', ''), - queue_row.get('has_pdf', False), - vault, - paths.get('zotero_dir') if 'zotero_dir' in paths else None, - ) - if not resolved_pdf: - meta['ocr_status'] = 'nopdf' - meta['error'] = 'PDF not found or not readable' - queue_row['queue_status'] = 'nopdf' - write_json(paths['ocr'] / key / 'meta.json', meta) - changed += 1 - continue - if not token: - meta['ocr_status'] = 'blocked' - meta['error'] = 'PaddleOCR not configured' - queue_row['queue_status'] = 'blocked' - write_json(paths['ocr'] / key / 'meta.json', meta) - changed += 1 - continue - try: - with open(resolved_pdf, 'rb') as file_handle: - response = requests.post(job_url, headers={'Authorization': f'bearer {token}'}, data={'model': model, 'optionalPayload': json.dumps(optional_payload)}, files={'file': file_handle}, timeout=120) - response.raise_for_status() - except Exception as e: - from paperforge.ocr_diagnostics import classify_error - state, suggestion = classify_error(e, getattr(e, 'response', None)) - meta['ocr_status'] = state - meta['error'] = str(e) - meta['suggestion'] = suggestion - queue_row['queue_status'] = state - write_json(paths['ocr'] / key / 'meta.json', meta) - changed += 1 - continue - meta['ocr_job_id'] = response.json()['data']['jobId'] - meta['ocr_status'] = 'queued' - meta['ocr_started_at'] = datetime.now(timezone.utc).isoformat() - meta['error'] = '' - queue_row['queue_status'] = 'queued' - write_json(paths['ocr'] / key / 'meta.json', meta) - changed += 1 - available_slots -= 1 - if queue_changed: - ocr_queue = [row for row in ocr_queue if str(row.get('queue_status', '')).lower() != 'done'] - write_ocr_queue(paths, ocr_queue) - run_selection_sync(vault) - run_index_refresh(vault) - print(f'ocr: updated {changed} records') - return 0 - -def _resolve_formal_note_path(vault: Path, zotero_key: str, domain: str) -> Path | None: - """Resolve formal literature note by zotero_key.""" - lit_root = pipeline_paths(vault)['literature'] - domain_dir = lit_root / domain - if not domain_dir.exists(): - return None - frontmatter_pattern = re.compile(f'^\\s*zotero_key:\\s*"?{re.escape(zotero_key)}"?\\s*$', re.MULTILINE) - for note_path in domain_dir.rglob('*.md'): - try: - text = note_path.read_text(encoding='utf-8') - except UnicodeDecodeError: - text = note_path.read_text(encoding='utf-8', errors='ignore') - if frontmatter_pattern.search(text): - return note_path - return None - -def run_deep_reading(vault: Path, verbose: bool = False) -> int: - """Sync deep-reading status between formal notes and library records. - - This worker does NOT generate content. It only: - 1. Scans formal literature notes for `## 🔍 精读` content - 2. Updates library-records/*.md frontmatter to match actual state - 3. Reports the queue of papers awaiting deep reading - - Actual content filling is done via /pf-deep (agent-driven). - """ - paths = pipeline_paths(vault) - config = load_domain_config(paths) - ensure_base_views(vault, paths, config) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} - synced = 0 - pending_queue: list[dict] = [] - for export_path in sorted(paths['exports'].glob('*.json')): - domain = domain_lookup.get(export_path.name, export_path.stem) - for item in load_export_rows(export_path): - key = item['key'] - record_dir = paths['library_records'] / domain - record_path = record_dir / f'{key}.md' - if not record_path.exists(): - continue - record_text = record_path.read_text(encoding='utf-8') - analyze_match = re.search('^analyze:\\s*(true|false)$', record_text, re.MULTILINE) - is_analyze = analyze_match and analyze_match.group(1) == 'true' - do_ocr_match = re.search('^do_ocr:\\s*(true|false)$', record_text, re.MULTILINE) - is_do_ocr = do_ocr_match and do_ocr_match.group(1) == 'true' - note_path = _resolve_formal_note_path(vault, key, domain) - has_content = False - if note_path and note_path.exists(): - note_text = note_path.read_text(encoding='utf-8') - has_content = has_deep_reading_content(note_text) - correct_status = 'done' if has_content else 'pending' - status_match = re.search('^deep_reading_status:\\s*"??"?$', record_text, re.MULTILINE) - current_status = status_match.group(1) if status_match else 'pending' - if current_status != correct_status: - new_text = re.sub('^deep_reading_status:\\s*"?.*?"?$', f'deep_reading_status: {yaml_quote(correct_status)}', record_text, flags=re.MULTILINE, count=1) - record_path.write_text(new_text, encoding='utf-8') - synced += 1 - if is_analyze and correct_status == 'pending': - meta_path = paths['ocr'] / key / 'meta.json' - ocr_status = 'pending' - if meta_path.exists(): - try: - meta = read_json(meta_path) - validated_status, error_msg = validate_ocr_meta(paths, meta) - ocr_status = validated_status - except Exception: - pass - pending_queue.append({ - 'zotero_key': key, - 'domain': domain, - 'title': item.get('title', ''), - 'ocr_status': ocr_status, - 'is_analyze': is_analyze, - 'is_do_ocr': is_do_ocr, - }) - if pending_queue: - ready = [q for q in pending_queue if q['ocr_status'] == 'done'] - waiting = [q for q in pending_queue if q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing')] - blocked = [q for q in pending_queue if q['is_analyze'] and q['ocr_status'] not in ('done', '') and not (q['is_do_ocr'] and q['ocr_status'] in ('pending', 'processing'))] - report_lines = ['# 待精读队列', ''] - if ready: - report_lines.extend([f'## 就绪 ({len(ready)} 篇) — OCR 已完成，可直接 /pf-deep', '']) - for q in ready: - report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']}") - report_lines.append('') - if waiting: - report_lines.extend([f'## 等待 OCR ({len(waiting)} 篇)', '']) - for q in waiting: - report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status']}") - report_lines.append('') - if blocked: - report_lines.extend([f'## 阻塞 ({len(blocked)} 篇) — 需要先完成 OCR', '']) - for q in blocked: - report_lines.append(f"- `{q['zotero_key']}` | {q['domain']} | {q['title']} | OCR: {q['ocr_status'] or '未启动'}") - report_lines.append('') - if verbose: - report_lines.append('### 修复步骤\n') - for q in blocked: - ocr_s = q['ocr_status'] or '' - if not ocr_s or ocr_s == 'pending': - fix = f"paperforge ocr" - report_lines.append(f"- `{q['zotero_key']}`: 运行 `{fix}` 启动 OCR") - elif ocr_s == 'processing': - report_lines.append(f"- `{q['zotero_key']}`: OCR 进行中，请等待完成") - elif ocr_s == 'failed': - report_lines.append(f"- `{q['zotero_key']}`: OCR 失败 — 检查 meta.json 错误信息，然后重新运行 `paperforge ocr`") - else: - report_lines.append(f"- `{q['zotero_key']}`: 运行 `paperforge ocr` 重试") - report_lines.append('') - report_lines.extend(['## 操作', '', '- 对就绪论文，使用 `/pf-deep ` 触发精读', '- 批量触发：提供多个 key，用 subagent 并行处理', '']) - else: - report_lines = ['# 待精读队列', '', '所有 analyze=true 的论文已完成精读。', ''] - report_path = paths['pipeline'] / 'deep-reading-queue.md' - report_path.write_text('\n'.join(report_lines), encoding='utf-8') - print(f'deep-reading: synced {synced} records, {len(pending_queue)} pending') - return 0 - -def _find_export_for_domain(paths: dict[str, Path], domain: str) -> Path | None: - """Find the BBT export JSON file for a given domain.""" - for export_path in sorted(paths["exports"].glob("*.json")): - if export_path.stem == domain: - return export_path - return None - - -def _detect_path_errors(paths: dict[str, Path], verbose: bool = False) -> dict: - """Scan library-records for path_error fields. - - Returns dict with: - - total: total count of records with path_error - - by_type: dict mapping error type -> count - - records: list of record dicts with keys: path, zotero_key, domain, path_error, bbt_path_raw - """ - result: dict = {"total": 0, "by_type": {}, "records": []} - if not paths["library_records"].exists(): - return result - for record_path in paths["library_records"].rglob("*.md"): - try: - text = record_path.read_text(encoding="utf-8") - except Exception as e: - if verbose: - print(f"[repair] error reading {record_path}: {e}") - continue - err_match = re.search(r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE) - if not err_match: - continue - path_error = err_match.group(1).strip() - if not path_error: - continue - key_match = re.search(r'^zotero_key:\s*"?(.+?)"?\s*$', text, re.MULTILINE) - if not key_match: - continue - zotero_key = key_match.group(1).strip() - domain = record_path.parent.name - bbt_match = re.search(r'^bbt_path_raw:\s*"(.*?)"\s*$', text, re.MULTILINE) - bbt_path_raw = bbt_match.group(1) if bbt_match else "" - result["total"] += 1 - result["by_type"][path_error] = result["by_type"].get(path_error, 0) + 1 - result["records"].append( - { - "path": record_path, - "zotero_key": zotero_key, - "domain": domain, - "path_error": path_error, - "bbt_path_raw": bbt_path_raw, - "text": text, - } - ) - return result - - -def repair_pdf_paths( - vault: Path, - paths: dict[str, Path], - error_records: list[dict], - verbose: bool = False, -) -> int: - """Re-resolve PDF paths for items with path_error. - - Returns number of paths successfully fixed. - """ - fixed = 0 - from paperforge.pdf_resolver import resolve_pdf_path - - cfg = load_vault_config(vault) - zotero_dir = vault / cfg.get("system_dir", "99_System") / "Zotero" - - # Cache export rows by domain to avoid reloading - domain_exports: dict[str, list[dict]] = {} - - for record in error_records: - record_path = record["path"] - zotero_key = record["zotero_key"] - domain = record["domain"] - path_error = record["path_error"] - text = record["text"] - - # For not_found errors, try to find the item in BBT export and re-process - if path_error == "not_found": - export_rows = domain_exports.get(domain) - if export_rows is None: - export_path = _find_export_for_domain(paths, domain) - if export_path and export_path.exists(): - try: - export_rows = load_export_rows(export_path) - domain_exports[domain] = export_rows - except Exception as e: - if verbose: - print( - f"[repair] error loading export for {domain}: {e}" - ) - export_rows = [] - else: - export_rows = [] - domain_exports[domain] = export_rows - - item = next((r for r in export_rows if r["key"] == zotero_key), None) - if item: - pdf_path = item.get("pdf_path", "") - if pdf_path: - new_wikilink = obsidian_wikilink_for_pdf( - pdf_path, vault, zotero_dir - ) - if new_wikilink: - new_text = update_frontmatter_field( - text, "pdf_path", new_wikilink - ) - new_text = update_frontmatter_field( - new_text, "path_error", "" - ) - new_text = update_frontmatter_field( - new_text, - "bbt_path_raw", - item.get("bbt_path_raw", ""), - ) - new_text = update_frontmatter_field( - new_text, - "zotero_storage_key", - item.get("zotero_storage_key", ""), - ) - if new_text != text: - record_path.write_text(new_text, encoding="utf-8") - fixed += 1 - if verbose: - print( - f"[repair] fixed path for {zotero_key}: {new_wikilink}" - ) - continue - - # For all errors, try resolving the current pdf_path - pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE) - if pdf_match: - current_pdf = pdf_match.group(1).strip() - if current_pdf: - raw_path = current_pdf.strip("[]") - resolved = resolve_pdf_path(raw_path, True, vault, zotero_dir) - if resolved: - new_text = update_frontmatter_field(text, "path_error", "") - if new_text != text: - record_path.write_text(new_text, encoding="utf-8") - fixed += 1 - if verbose: - print(f"[repair] cleared path_error for {zotero_key}") - else: - if verbose: - print(f"[repair] {zotero_key} path still unresolved") - else: - if verbose: - print( - f"[repair] {zotero_key} has empty pdf_path (not_found)" - ) - else: - if verbose: - print(f"[repair] {zotero_key} has no pdf_path field") - - return fixed - - -def run_repair(vault: Path, paths: dict, verbose: bool = False, fix: bool = False, fix_paths: bool = False) -> dict: - """Scan all domains for three-way state divergence and optionally repair. - - Compares three sources of ocr_status: - 1. library_record.md frontmatter ocr_status - 2. formal_note.md frontmatter ocr_status - 3. meta.json ocr_status (post-validate_ocr_meta()) - - Returns: - dict with scanned, divergent, fixed, errors counts - """ - result = {"scanned": 0, "divergent": [], "fixed": 0, "errors": []} - config = load_domain_config(paths) - domain_lookup = {entry['export_file']: entry['domain'] for entry in config['domains']} - record_paths = list(paths['library_records'].rglob('*.md')) - for record_path in record_paths: - try: - record_text = record_path.read_text(encoding='utf-8') - except Exception as e: - result['errors'].append({"file": str(record_path), "error": str(e)}) - continue - key_match = re.search('^zotero_key:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE) - if not key_match: - continue - zotero_key = key_match.group(1).strip() - domain = record_path.parent.name - record_dir = record_path.parent - result['scanned'] += 1 - lib_ocr_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', record_text, re.MULTILINE) - lib_ocr_status = lib_ocr_match.group(1).strip() if lib_ocr_match else 'pending' - note_path = _resolve_formal_note_path(vault, zotero_key, domain) - note_ocr_status = None - if note_path and note_path.exists(): - try: - note_text = note_path.read_text(encoding='utf-8') - note_status_match = re.search('^ocr_status:\\s*"?(.+?)"?\\s*$', note_text, re.MULTILINE) - note_ocr_status = note_status_match.group(1).strip() if note_status_match else None - except Exception: - pass - meta_path = paths['ocr'] / zotero_key / 'meta.json' - meta_ocr_status = None - meta_validated_status = None - validated_status = None - validated_error = "" - if meta_path.exists(): - try: - meta = read_json(meta_path) - validated_status, validated_error = validate_ocr_meta(paths, meta) - meta_validated_status = validated_status - if validated_error and verbose: - print(f"[repair] {zotero_key} meta validation error: {validated_error}") - raw_status = str(meta.get('ocr_status', '') or '').strip().lower() - meta_ocr_status = raw_status if raw_status else None - if meta_validated_status == 'done_incomplete': - meta_ocr_status = 'done_incomplete' - except Exception as e: - result['errors'].append({"file": str(meta_path), "error": str(e)}) - meta_ocr_status = None - is_divergent = False - div_reason = "" - if meta_validated_status == 'done_incomplete': - is_divergent = True - div_reason = f"meta validation: done_incomplete ({validated_error})" - elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing', None): - is_divergent = True - div_reason = f"library_record done but meta {meta_ocr_status or 'missing'}" - elif note_ocr_status == 'done' and (meta_ocr_status is None or meta_validated_status == 'done_incomplete'): - is_divergent = True - div_reason = "formal_note done but meta.json missing/invalid" - elif lib_ocr_status != 'pending' and meta_ocr_status is not None and meta_validated_status is not None and lib_ocr_status != meta_validated_status: - is_divergent = True - div_reason = f"library_record={lib_ocr_status} vs meta post-validation={meta_validated_status}" - if is_divergent: - item = { - "zotero_key": zotero_key, - "domain": domain, - "library_record_ocr_status": lib_ocr_status, - "formal_note_ocr_status": note_ocr_status, - "meta_ocr_status": meta_validated_status or meta_ocr_status, - "reason": div_reason, - } - result['divergent'].append(item) - if verbose: - print(f"[repair] divergent: {zotero_key} | {div_reason}") - if fix: - fixed_library_record = False - fixed_formal_note = False - fixed_meta = False - new_status = 'pending' - if meta_ocr_status is None or meta_validated_status == 'done_incomplete': - new_status = 'pending' - new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status) - if new_record_text != record_text: - record_path.write_text(new_record_text, encoding='utf-8') - fixed_library_record = True - if note_path and note_path.exists(): - try: - note_text = note_path.read_text(encoding='utf-8') - new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status) - if new_note_text != note_text: - note_path.write_text(new_note_text, encoding='utf-8') - fixed_formal_note = True - except Exception: - pass - if meta_validated_status is not None and meta_validated_status != 'done': - if meta_path.exists(): - try: - meta = read_json(meta_path) - meta['ocr_status'] = 'pending' - write_json(meta_path, meta) - fixed_meta = True - except Exception: - pass - record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE) - is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true' - if not is_do_ocr: - final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true') - if final_record_text != new_record_text: - record_path.write_text(final_record_text, encoding='utf-8') - fixed_library_record = True - elif lib_ocr_status == 'done' and meta_ocr_status in ('pending', 'processing'): - new_status = 'pending' - new_record_text = update_frontmatter_field(record_text, 'ocr_status', new_status) - if new_record_text != record_text: - record_path.write_text(new_record_text, encoding='utf-8') - fixed_library_record = True - if note_path and note_path.exists(): - try: - note_text = note_path.read_text(encoding='utf-8') - new_note_text = update_frontmatter_field(note_text, 'ocr_status', new_status) - if new_note_text != note_text: - note_path.write_text(new_note_text, encoding='utf-8') - fixed_formal_note = True - except Exception: - pass - if meta_path.exists(): - try: - meta = read_json(meta_path) - meta['ocr_status'] = 'pending' - write_json(meta_path, meta) - fixed_meta = True - except Exception: - pass - record_do_ocr_match = re.search(r'^do_ocr:\s*(true|false)$', new_record_text, re.MULTILINE) - is_do_ocr = record_do_ocr_match and record_do_ocr_match.group(1) == 'true' - if not is_do_ocr: - final_record_text = update_frontmatter_field(new_record_text, 'do_ocr', 'true') - if final_record_text != new_record_text: - record_path.write_text(final_record_text, encoding='utf-8') - fixed_library_record = True - fixed_count = sum([fixed_library_record, fixed_formal_note, fixed_meta]) - result['fixed'] += fixed_count - if verbose and fixed_count > 0: - print(f"[repair] fixed {fixed_count} files for {zotero_key}") - # Path error detection and repair - path_errors = _detect_path_errors(paths, verbose) - if path_errors["total"] > 0: - error_summary = ", ".join( - f"{count} {err}" for err, count in sorted(path_errors["by_type"].items()) - ) - print( - f"[repair] Found {path_errors['total']} items with path errors: {error_summary}" - ) - if fix_paths: - fixed_count = repair_pdf_paths( - vault, paths, path_errors["records"], verbose - ) - print(f"[repair] Fixed {fixed_count} PDF paths") - else: - print("[repair] Tip: run with --fix-paths to attempt auto-resolution") - elif verbose: - print("[repair] No path errors found") - - result["path_errors"] = path_errors - return result - -def _detect_zotero_data_dir() -> str | None: - """Try to detect the user's Zotero data directory.""" - if os.name == "nt": - appdata = os.environ.get("APPDATA", "") - if appdata: - candidate = Path(appdata) / "Zotero" - if candidate.exists() and (candidate / "storage").exists(): - return str(candidate) - home = Path.home() - candidates = [] - if os.name == "nt": - candidates = [ - home / "Zotero", - home / "AppData" / "Roaming" / "Zotero", - ] - else: - candidates = [ - home / "Zotero", - home / ".zotero" / "zotero", - home / "Library" / "Application Support" / "Zotero", - ] - for candidate in candidates: - if candidate.exists() and (candidate / "storage").exists(): - return str(candidate) - return None - - -def check_zotero_location(vault: Path, cfg: dict, add_check) -> None: - """Detect if Zotero data directory is inside vault or linked via junction.""" - system_dir = vault / cfg["system_dir"] - zotero_link = system_dir / "Zotero" - if zotero_link.exists(): - if zotero_link.is_symlink() or _is_junction(zotero_link): - try: - target = os.path.realpath(zotero_link) - if Path(target).exists(): - add_check( - "Path Resolution", - "pass", - f"Zotero junction valid -> {target}", - ) - else: - add_check( - "Path Resolution", - "warn", - f"Zotero junction target missing: {target}", - f"Recreate junction: mklink /J \"{zotero_link}\" \"\"", - ) - except Exception: - add_check( - "Path Resolution", - "warn", - "Zotero junction exists but target could not be resolved", - ) - else: - if (zotero_link / "storage").exists(): - add_check( - "Path Resolution", - "pass", - "Zotero inside vault -- direct paths available", - ) - else: - add_check( - "Path Resolution", - "warn", - "Zotero directory exists but missing storage/ subdirectory", - "Verify this is a valid Zotero data directory", - ) - else: - zotero_data_dir = _detect_zotero_data_dir() - if zotero_data_dir: - add_check( - "Path Resolution", - "warn", - "Zotero outside vault -- junction recommended", - f'Run as Administrator: mklink /J "{zotero_link}" "{zotero_data_dir}"', - ) - else: - add_check( - "Path Resolution", - "fail", - "Zotero directory not found", - f'Run as Administrator: mklink /J "{zotero_link}" ""', - ) - - -def _summarize_errors(errors: list[str]) -> str: - """Summarize error types into a human-readable string.""" - counts: dict[str, int] = {} - for e in errors: - counts[e] = counts.get(e, 0) + 1 - return ", ".join(f"{count} {err}" for err, count in sorted(counts.items())) - - -def check_pdf_paths(vault: Path, paths: dict, add_check) -> None: - """Sample up to 5 library records and validate their pdf_path wikilinks.""" - record_paths = list(paths["library_records"].rglob("*.md")) - if not record_paths: - add_check("Path Resolution", "pass", "No library records to check") - return - import random - - sample = ( - record_paths if len(record_paths) <= 5 else random.sample(record_paths, 5) - ) - valid = 0 - errors: list[str] = [] - for record_path in sample: - try: - text = record_path.read_text(encoding="utf-8") - except Exception: - continue - pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE) - if not pdf_match: - continue - pdf_path = pdf_match.group(1).strip() - if not pdf_path: - continue - raw_path = pdf_path.strip("[]") - candidate = vault / raw_path.replace("/", os.sep) - if candidate.exists(): - valid += 1 - else: - err_match = re.search( - r'^path_error:\s*"(.*?)"\s*$', text, re.MULTILINE - ) - error_type = err_match.group(1) if err_match else "not_found" - errors.append(error_type) - total = len(sample) - if valid == total: - add_check("Path Resolution", "pass", f"{valid}/{total} PDF paths valid") - else: - error_summary = _summarize_errors(errors) if errors else "unknown" - add_check( - "Path Resolution", - "warn", - f"{valid}/{total} PDF paths valid, {total - valid} path errors: {error_summary}", - ) - - -def check_wikilink_format(vault: Path, paths: dict, add_check) -> None: - """Verify all pdf_path values in library-records use [[...]] format.""" - record_paths = list(paths["library_records"].rglob("*.md")) - bad_paths: list[str] = [] - for record_path in record_paths: - try: - text = record_path.read_text(encoding="utf-8") - except Exception: - continue - pdf_match = re.search(r'^pdf_path:\s*"(.*?)"\s*$', text, re.MULTILINE) - if not pdf_match: - continue - pdf_path = pdf_match.group(1).strip() - if not pdf_path: - continue - if not (pdf_path.startswith("[[") and pdf_path.endswith("]]")): - bad_paths.append(str(record_path.relative_to(vault))) - if bad_paths: - add_check( - "Path Resolution", - "warn", - f"{len(bad_paths)} pdf_path values not in wikilink format", - "Re-run `paperforge sync` to regenerate wikilinks", - ) - else: - add_check( - "Path Resolution", - "pass", - "All pdf_path values use wikilink format", - ) - - -def run_doctor(vault: Path) -> int: - """Validate PaperForge Lite setup and report by category. - - Returns: - 0 if all checks pass, 1 otherwise. - """ - paths = pipeline_paths(vault) - cfg = load_vault_config(vault) - checks: list[tuple[str, str, str, str]] = [] - - def add_check(category: str, status: str, message: str, fix: str = "") -> None: - checks.append((category, status, message, fix)) - - sys_version = sys.version_info - if sys_version.major >= 3 and sys_version.minor >= 10: - add_check("Python 环境", "pass", f"Python {sys_version.major}.{sys_version.minor}.{sys_version.micro}") - else: - add_check("Python 环境", "fail", f"Python {sys_version.major}.{sys_version.minor} (需要 3.10+)", "升级 Python 到 3.10 或更高版本") - - required_modules = ["requests", "pymupdf", "PIL", "yaml"] - missing_modules = [] - for mod in required_modules: - try: - __import__(mod) - except ImportError: - missing_modules.append(mod) - if missing_modules: - add_check("Python 环境", "fail", f"缺少模块: {', '.join(missing_modules)}", f"运行: pip install {' '.join(missing_modules)}") - elif sys_version.major >= 3 and sys_version.minor >= 10: - pass - - if (vault / "paperforge.json").exists(): - add_check("Vault 结构", "pass", "paperforge.json 存在") - else: - add_check("Vault 结构", "fail", "paperforge.json 不存在", "在 vault 根目录创建 paperforge.json") - - system_dir = vault / cfg["system_dir"] - if system_dir.exists(): - add_check("Vault 结构", "pass", f"system_dir 存在: {cfg['system_dir']}") - else: - add_check("Vault 结构", "fail", f"system_dir 不存在: {cfg['system_dir']}", f"运行: mkdir {cfg['system_dir']}") - - resources_dir = vault / cfg["resources_dir"] - if resources_dir.exists(): - add_check("Vault 结构", "pass", f"resources_dir 存在: {cfg['resources_dir']}") - else: - add_check("Vault 结构", "fail", f"resources_dir 不存在: {cfg['resources_dir']}", f"运行: mkdir {cfg['resources_dir']}") - - control_dir = resources_dir / cfg.get("control_dir", "LiteratureControl") - if control_dir.exists(): - add_check("Vault 结构", "pass", f"control_dir 存在: {cfg.get('control_dir', 'LiteratureControl')}") - else: - add_check("Vault 结构", "fail", f"control_dir 不存在: {cfg.get('control_dir', 'LiteratureControl')}", f"运行: mkdir {cfg['resources_dir']}/{cfg.get('control_dir', 'LiteratureControl')}") - - zotero_link = system_dir / "Zotero" - if zotero_link.exists(): - if zotero_link.is_symlink() or _is_junction(zotero_link): - add_check("Zotero 链接", "pass", "Zotero 目录是 junction/symlink") - else: - add_check("Zotero 链接", "warn", "Zotero 目录存在但不是 junction，建议使用 junction") - else: - add_check("Zotero 链接", "fail", "Zotero 目录不存在", f"创建 junction: mklink /j {zotero_link} ") - - exports_dir = system_dir / "PaperForge" / "exports" - if exports_dir.exists(): - add_check("BBT 导出", "pass", "exports 目录存在") - else: - add_check("BBT 导出", "fail", "exports 目录不存在", "在 Better BibTeX 设置中配置导出路径") - - # Check for any valid JSON export (per-domain or library.json) - json_files = sorted(exports_dir.glob("*.json")) if exports_dir.exists() else [] - valid_exports = [] - for jf in json_files: - try: - data = json.loads(jf.read_text(encoding="utf-8")) - if isinstance(data, list) and len(data) > 0: - has_key = any("key" in item or "citation-key" in item for item in data[:5]) - if has_key: - valid_exports.append((jf.name, len(data))) - elif isinstance(data, dict) and isinstance(data.get("items"), list) and len(data["items"]) > 0: - has_key = any("key" in item or "citation-key" in item for item in data["items"][:5]) - if has_key: - valid_exports.append((jf.name, len(data["items"]))) - except (JSONDecodeError, Exception): - pass - - if valid_exports: - for name, count in valid_exports: - add_check("BBT 导出", "pass", f"{name} 正常 ({count} 条)") - elif json_files: - add_check("BBT 导出", "warn", "导出文件存在但无有效 citation key") - else: - add_check("BBT 导出", "fail", "未找到 JSON 导出文件", "在 Zotero Better BibTeX 设置中配置导出路径") - - env_api_key = os.environ.get("PADDLEOCR_API_TOKEN") or os.environ.get("PADDLEOCR_API_KEY") or os.environ.get("OCR_TOKEN") - if env_api_key: - add_check("OCR 配置", "pass", "API Token 已配置") - else: - add_check("OCR 配置", "fail", "缺少 PADDLEOCR_API_TOKEN", "在 .env 文件中设置 PADDLEOCR_API_TOKEN") - - if (paths["pipeline"] / "literature_pipeline.py").exists(): - add_check("Worker 脚本", "pass", "literature_pipeline.py 存在") - try: - from pipeline.worker.scripts.literature_pipeline import ( - run_status, run_selection_sync, run_index_refresh, - run_deep_reading, run_ocr, ensure_base_views, - ) - add_check("Worker 脚本", "pass", "所有 worker 函数可导入") - except ImportError as e: - add_check("Worker 脚本", "fail", f"worker 函数导入失败: {e}", "检查 pipeline/worker/scripts/literature_pipeline.py") - else: - add_check("Worker 脚本", "fail", "literature_pipeline.py 不存在", "确认 PaperForge pipeline 已正确安装") - - # Path Resolution checks - check_zotero_location(vault, cfg, add_check) - check_pdf_paths(vault, paths, add_check) - check_wikilink_format(vault, paths, add_check) - - ld_deep_script = paths.get("ld_deep_script") - skill_dir = None - if ld_deep_script: - skill_dir = ld_deep_script.parent.parent - if skill_dir and skill_dir.exists(): - # Try actual importability check - ld_deep_import_ok = False - import_error = "" - if ld_deep_script and ld_deep_script.exists(): - try: - import importlib.util - spec = importlib.util.spec_from_file_location("ld_deep", ld_deep_script) - if spec and spec.loader: - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - ld_deep_import_ok = True - except Exception as e: - import_error = str(e) - if ld_deep_import_ok: - add_check("Agent 脚本", "pass", "paperforge and ld_deep importable") - else: - add_check("Agent 脚本", "warn", f"literature-qa skill 目录存在但 import 失败: {import_error}", "确认 agent_config_dir 配置正确并已运行 pip install -e .") - else: - add_check("Agent 脚本", "warn", "literature-qa skill 目录未找到", "确认 agent_config_dir 配置正确") - - print("PaperForge Lite Doctor") - print("=" * 40) - current_category = "" - fix_map: dict[str, list[str]] = {} - for category, status, message, fix in checks: - if category != current_category: - if current_category: - print() - current_category = category - status_tag = {"pass": "[PASS]", "fail": "[FAIL]", "warn": "[WARN]"}[status] - print(f"{status_tag} {category} — {message}") - if status == "fail" and fix: - fix_map.setdefault(category, []) - fix_map[category].append(fix) - - if fix_map: - print("\n修复步骤:") - for cat, fixes in fix_map.items(): - for f in fixes: - print(f" - {cat}: {f}") - print() - return 1 - print() - return 0 - - -def _is_junction(path: Path) -> bool: - """Check if a path is a Windows junction point.""" - try: - import ctypes - from ctypes import wintypes - FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 - FILE_ATTRIBUTE_REPARSE_POINT = 0x400 - INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF - GetFileAttributesW = ctypes.windll.kernel32.GetFileAttributesW - GetFileAttributesW.argtypes = [wintypes.LPCWSTR] - GetFileAttributesW.restype = wintypes.DWORD - attrs = GetFileAttributesW(str(path)) - if attrs == INVALID_FILE_ATTRIBUTES: - return False - return bool(attrs & FILE_ATTRIBUTE_REPARSE_POINT) - except Exception: - return False - - -def run_status(vault: Path) -> int: - """Print a compact Lite install/runtime status.""" - paths = pipeline_paths(vault) - cfg = load_vault_config(vault) - config = load_domain_config(paths) - ensure_base_views(vault, paths, config) - export_files = sorted(paths['exports'].glob('*.json')) - record_count = sum(1 for _ in paths['library_records'].rglob('*.md')) if paths['library_records'].exists() else 0 - note_count = sum(1 for _ in paths['literature'].rglob('*.md')) if paths['literature'].exists() else 0 - base_count = sum(1 for _ in paths['bases'].glob('*.base')) if paths['bases'].exists() else 0 - ocr_done = 0 - ocr_total = 0 - if paths['ocr'].exists(): - for meta_path in paths['ocr'].glob('*/meta.json'): - ocr_total += 1 - try: - meta = read_json(meta_path) - except Exception: - continue - if str(meta.get('ocr_status', '')).strip().lower() == 'done': - ocr_done += 1 - env_paths = [vault / '.env', paths['pipeline'] / '.env'] - env_found = [str(path.relative_to(vault)).replace('\\', '/') for path in env_paths if path.exists()] - - # Count path errors - path_error_count = 0 - if paths['library_records'].exists(): - for record_path in paths['library_records'].rglob('*.md'): - try: - text = record_path.read_text(encoding='utf-8') - if re.search(r'^path_error:\s*"(.+?)"\s*$', text, re.MULTILINE): - path_error_count += 1 - except Exception: - continue - - print('PaperForge Lite status') - print(f"- vault: {vault}") - print(f"- system_dir: {cfg['system_dir']}") - print(f"- resources_dir: {cfg['resources_dir']}") - print(f"- literature_dir: {cfg['literature_dir']}") - print(f"- control_dir: {cfg['control_dir']}") - print(f"- exports: {len(export_files)} JSON file(s)") - print(f"- domains: {len(config.get('domains', []))}") - print(f"- library_records: {record_count}") - print(f"- formal_notes: {note_count}") - print(f"- bases: {base_count}") - print(f"- ocr: {ocr_done}/{ocr_total} done") - print(f"- path_errors: {path_error_count}") - if path_error_count > 0: - print(" Tip: Run `paperforge repair --fix-paths` to attempt resolution") - print(f"- env: {', '.join(env_found) if env_found else 'not configured'}") - return 0 - -# ============================================================================= -# Update 功能 -# ============================================================================= - -GITHUB_REPO = "LLLin000/PaperForge" -GITHUB_ZIP = f"https://github.com/{GITHUB_REPO}/archive/refs/heads/master.zip" - -UPDATEABLE_PATHS = ["skills", "pipeline", "command", "scripts"] - - -def protected_paths(vault: Path) -> set[str]: - cfg = load_vault_config(vault) - pf = f"{cfg['system_dir']}/PaperForge" - return { - cfg["resources_dir"], - cfg["base_dir"], - f"{pf}/ocr", - f"{pf}/exports", - f"{pf}/indexes", - f"{pf}/candidates", - ".env", - "AGENTS.md", - } - - -def _color(text: str, c: str = "") -> str: - colors = {"r": "\033[91m", "g": "\033[92m", "y": "\033[93m", "b": "\033[94m", "c": "\033[96m", "x": "\033[0m"} - if sys.platform == "win32" and not os.environ.get("FORCE_COLOR"): - return text - return f"{colors.get(c, '')}{text}{colors['x']}" - - -def _log(msg: str, c: str = "") -> None: - print(_color(msg, c)) - - -def _remote_version() -> str | None: - try: - api = f"https://api.github.com/repos/{GITHUB_REPO}/contents/paperforge.json" - req = urllib.request.Request(api, headers={"Accept": "application/vnd.github.v3+json", "User-Agent": "PaperForge"}) - with urllib.request.urlopen(req, timeout=10) as resp: - data = json.loads(resp.read()) - req2 = urllib.request.Request(data["download_url"], headers={"User-Agent": "PaperForge"}) - with urllib.request.urlopen(req2, timeout=10) as resp2: - return json.loads(resp2.read()).get("version") - except Exception: - return None - - -def _scan_updates(vault: Path, source: Path) -> list[tuple[Path, Path, str]]: - updates = [] - protected = protected_paths(vault) - for name in UPDATEABLE_PATHS: - src_dir = source / name - if not src_dir.exists(): - continue - for src in src_dir.rglob("*"): - if not src.is_file(): - continue - rel = src.relative_to(source) - dst = vault / rel - rel_str = str(rel).replace("\\", "/") - if any(rel_str.startswith(p) for p in protected): - continue - if dst.exists(): - if hashlib.sha256(src.read_bytes()).hexdigest() != hashlib.sha256(dst.read_bytes()).hexdigest(): - updates.append((src, dst, "UPDATE")) - else: - updates.append((src, dst, "NEW")) - return updates - - -def _do_backup(vault: Path, updates: list) -> Path | None: - backup_dir = vault / f".backup_{datetime.now():%Y%m%d_%H%M%S}" - backup_dir.mkdir(exist_ok=True) - count = 0 - for src, dst, action in updates: - if action == "UPDATE" and dst.exists(): - bp = backup_dir / dst.relative_to(vault) - bp.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(dst, bp) - count += 1 - if count: - _log(f"[INFO] 已备份 {count} 个文件到 {backup_dir.name}", "c") - return backup_dir if count else None - - -def _apply_updates(vault: Path, updates: list) -> bool: - try: - for src, dst, action in updates: - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dst) - return True - except Exception as e: - _log(f"[ERR] 更新失败: {e}", "r") - return False - - -def _rollback(vault: Path, backup_dir: Path) -> None: - _log("[INFO] 正在回滚...", "b") - for bp in backup_dir.rglob("*"): - if bp.is_file(): - orig = vault / bp.relative_to(backup_dir) - orig.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(bp, orig) - _log("[OK] 回滚完成", "g") - - -def update_via_git(vault: Path) -> bool: - if not (vault / ".git").is_dir(): - _log("[ERR] 不是 git 仓库", "r") - return False - r = subprocess.run(["git", "status", "--short"], cwd=vault, capture_output=True, text=True, encoding="utf-8") - if r.stdout.strip(): - _log("[WARN] 有未提交的更改，请先提交或储藏", "y") - return False - _log("[INFO] 执行 git pull...", "b") - r = subprocess.run(["git", "pull", "origin", "master"], cwd=vault, capture_output=True, text=True, encoding="utf-8") - if r.returncode != 0: - _log(f"[ERR] git pull 失败: {r.stderr}", "r") - return False - _log("[OK] git pull 成功", "g") - if r.stdout.strip(): - print(r.stdout) - return True - - -def update_via_zip(vault: Path) -> bool: - _log("[INFO] 下载更新包...", "b") - tmp = Path(tempfile.mkdtemp(prefix="pf_update_")) - zip_path = tmp / "update.zip" - try: - req = urllib.request.Request(GITHUB_ZIP, headers={"User-Agent": "PaperForge"}) - with urllib.request.urlopen(req, timeout=60) as resp: - zip_path.write_bytes(resp.read()) - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(tmp / "extracted") - dirs = [d for d in (tmp / "extracted").iterdir() if d.is_dir()] - source = dirs[0] if dirs else None - if not source: - _log("[ERR] 解压失败", "r") - return False - updates = _scan_updates(vault, source) - if not updates: - _log("[OK] 所有文件已是最新", "g") - return True - _log(f"\n[INFO] 发现 {len(updates)} 个文件需要更新:", "b") - for src, dst, action in updates: - _log(f" [{action}] {dst.relative_to(vault)}", "g" if action == "NEW" else "y") - backup = _do_backup(vault, updates) - if _apply_updates(vault, updates): - _log(f"\n[OK] 更新完成！共 {len(updates)} 个文件", "g") - return True - if backup: - _rollback(vault, backup) - return False - except Exception as e: - _log(f"[ERR] 下载失败: {e}", "r") - return False - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def run_update(vault: Path) -> int: - """运行更新检查与安装""" - local_cfg = vault / "paperforge.json" - if not local_cfg.exists(): - _log("[ERR] 未找到 paperforge.json", "r") - return 1 - local = json.loads(local_cfg.read_text(encoding="utf-8")).get("version", "unknown") - remote = _remote_version() - _log("=" * 50, "b") - _log("PaperForge Lite 更新", "b") - _log("=" * 50, "b") - _log(f"本地版本: {local}", "c") - _log(f"远程版本: {remote or 'unknown'}", "c") - if not remote: - _log("[ERR] 无法获取远程版本", "r") - return 1 - try: - needs = tuple(int(x) for x in remote.split(".") if x.isdigit()) > tuple(int(x) for x in local.split(".") if x.isdigit()) - except ValueError: - needs = remote != local - if not needs: - _log("[OK] 当前已是最新版本", "g") - return 0 - _log(f"\n[INFO] 发现新版本: {local} -> {remote}", "y") - _log("[WARN] 更新前建议备份 Vault", "y") - ans = input(_color("确认更新? [y/N]: ", "y")).strip().lower() - if ans not in ("y", "yes"): - _log("[INFO] 已取消", "c") - return 0 - if (vault / ".git").is_dir(): - success = update_via_git(vault) - else: - success = update_via_zip(vault) - if success: - _log("\n[OK] 更新完成！请重启 Obsidian", "g") - return 0 if success else 1 - - -# ============================================================================= -# Main -# ============================================================================= - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument('--vault', required=True, type=Path) - parser.add_argument('--query') - parser.add_argument('--domain') - parser.add_argument('--recommended-collection', default='') - parser.add_argument('--requester-skill', default='') - parser.add_argument('--request-context', default='') - parser.add_argument('--recommend-reason', default='') - parser.add_argument('--limit', type=int, default=8) - parser.add_argument('--sources', nargs='+') - parser.add_argument('--skip-ingest', action='store_true') - parser.add_argument('worker', choices=['selection-sync', 'index-refresh', 'ocr', 'deep-reading', 'status', 'update', 'wizard', 'all']) - args = parser.parse_args() - paths = pipeline_paths(args.vault) - # Load .env from vault root first, then from the configured PaperForge directory. - load_simple_env(args.vault / '.env') - load_simple_env(paths['pipeline'] / '.env') - if args.worker == 'selection-sync': - return run_selection_sync(args.vault) - if args.worker == 'index-refresh': - return run_index_refresh(args.vault) - if args.worker == 'ocr': - return run_ocr(args.vault) - if args.worker == 'deep-reading': - return run_deep_reading(args.vault) - if args.worker == 'status': - return run_status(args.vault) - if args.worker == 'update': - return run_update(args.vault) - if args.worker == 'wizard': - wizard_script = args.vault / 'setup_wizard.py' - if not wizard_script.exists(): - print(f"[ERR] 未找到 {wizard_script}") - return 1 - return subprocess.run([sys.executable, str(wizard_script), '--vault', str(args.vault)]).returncode - # 'all' - 依次运行所有 Lite 版 workers - code = run_selection_sync(args.vault) - if code: - return code - code = run_index_refresh(args.vault) - if code: - return code - code = run_ocr(args.vault) - if code: - return code - return run_deep_reading(args.vault) -if __name__ == '__main__': - raise SystemExit(main()) diff --git a/skills/literature-qa/chart-reading/GSEA富集图.md b/skills/literature-qa/chart-reading/GSEA富集图.md deleted file mode 100644 index f7885bcf..00000000 --- a/skills/literature-qa/chart-reading/GSEA富集图.md +++ /dev/null @@ -1,78 +0,0 @@ -# GSEA 富集图 - -## 适用场景 - -展示基因集富集分析（Gene Set Enrichment Analysis）结果，说明某一基因集合（如通路、功能类别）在排序后的基因列表中是否富集于顶端或底端。常用于转录组、蛋白质组差异表达下游分析。 - -## 读图维度 - -### 1. 排序指标与排序逻辑 - -- **基因排序依据**： - - Signal2Noise ratio？logFC？t-statistic？Correlation？ - - 排序方向：高表达→低表达？病例→对照？ -- **排序方向含义**： - - 左侧（顶端）= 在 Case 中高表达 / 正相关 - - 右侧（底端）= 在 Control 中高表达 / 负相关 - -### 2. 富集曲线 (Enrichment Plot) - -- **曲线形态**： - - 高峰靠左 = 基因集富集于排序列表顶端（Case-associated） - - 高峰靠右 = 富集于底端（Control-associated） - - 平坦 = 无显著富集 -- **峰值位置**： - - 峰值在哪个分位数？（ES 最大点） - - 峰值是否陡峭？（富集程度强 vs 弱） - -### 3. 核心统计量 - -- **ES (Enrichment Score)**： - - 标准化 ES (NES) 范围通常在 -3 到 +3 - - |NES| > 1 通常被认为有意义（软件默认阈值） - - 但作者使用的实际阈值是什么？ -- **p-value / FDR (q-value)**： - - q < 0.05 还是 q < 0.25？（GSEA 默认用较宽松的 0.25） - - 是否经过多重检验校正？ -- **Leading Edge**： - - 哪些基因贡献了主要富集信号？ - - Leading edge 基因数 / 基因集总基因数 = ? - -### 4. 基因位置分布 - -- **中间条的竖线**： - - 每个竖线代表该基因集中的基因在排序列表中的位置 - - 竖线是否集中在左侧/右侧？还是均匀分布？ -- **基因集大小**： - - 总基因数是否在合理范围（15-500）？ - - 过小（<15）或过大（>500）的基因集结果不可靠 - -### 5. 热图条带 - -- **热图含义**： - - 通常表示所有基因的表达谱（红=高表达，蓝=低表达） - - 或表示基因与表型的相关性 -- **观察要点**： - - 热图颜色在 Leading Edge 区域是否一致？ - - 是否呈现清晰的分层？ - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **宽松FDR阈值** | q<0.25 作为显著性 | 大量假阳性通路，需要 q<0.05 | -| **基因集大小不当** | 看基因集总基因数 | 过小缺乏统计力，过大失去特异性 | -| **排序指标选择偏差** | 检查 ranking metric | 不同指标可能产生不同结果 | -| **Permutation次数不足** | nPerm < 1000 | p值不稳定，结果不可重复 | -| **多重数据库扫描** | 同时测试数百个基因集 | 即使 q-value 校正，仍可能过拟合 | - -## 关键问题模板 - -1. "GSEA 使用 q<0.25 作为阈值，如果收紧到 q<0.05，该通路是否仍然显著？" -2. "富集曲线峰值靠左，但 Leading Edge 基因只占基因集的 20%，富集信号是否主要由少数基因驱动？" -3. "排序使用的是 Signal2Noise，如果改用 logFC 或 t-statistic，该通路是否仍然富集？" -4. "该基因集包含 400+ 基因，是否过大导致失去生物学特异性？" - -## 一句话总结 - -> GSEA 图的核心是**"看排序、看曲线、看Leading Edge、防宽松阈值"**——富集曲线的峰值位置和陡峭度比单纯的 p 值更能说明问题。 diff --git a/skills/literature-qa/chart-reading/ROC与PR曲线.md b/skills/literature-qa/chart-reading/ROC与PR曲线.md deleted file mode 100644 index ce4023a8..00000000 --- a/skills/literature-qa/chart-reading/ROC与PR曲线.md +++ /dev/null @@ -1,66 +0,0 @@ -# ROC 与 PR 曲线阅读指南 - -## 适用场景 -- 评估分类/诊断模型的性能 -- 比较不同模型/特征的区分能力 -- 选择最优分类阈值 - ---- - -## 核心阅读维度 - -### 1. AUC / AUROC -- **数值解读**： - - 0.5 = 随机猜测（对角线） - - 0.7-0.8 = 可接受 - - 0.8-0.9 = 优秀 - - >0.9 = 杰出（但也可能是过拟合信号） -- **置信区间**：是否报告了95% CI？CI窄 = 估计精确；CI宽 = 样本小或不稳定 -- **与基线比较**：是否优于现有方法？差异是否有统计学意义？（DeLong检验） - -### 2. PR-AUC（Precision-Recall） -- **什么时候更重要**：当正负样本极度不平衡时（如罕见病诊断），PR曲线比ROC更有信息量 -- **基线**：PR曲线的基线 = 正样本比例（如正样本5%，则基线AP=0.05） -- **解读**：AP（Average Precision）> 基线多少？AP高但ROC AUC低 = 模型对正样本识别好，但假阳性多 - -### 3. 最优截断点 -- **Youden指数**：Sensitivity + Specificity - 1 的最大值点 -- **临床场景决定**： - - 筛查（如癌症早筛）：优先高Sensitivity（不漏诊），容忍低Specificity - - 确诊（如手术决策）：优先高Specificity（不误诊），容忍低Sensitivity -- **作者的选择**：是否根据临床需求选择了截断点？还是只用默认0.5？ - -### 4. 曲线形态 -- **ROC左上角凸起**：越好（高Sensitivity + 高Specificity同时达到） -- **PR右上角凸起**：越好（高Precision + 高Recall同时达到） -- **交叉曲线**：两条曲线交叉 = 一个模型在某些阈值更好，另一个在其他阈值更好 -- **平台期**：曲线在某段平台 = 改变阈值对该区间性能影响小 - -### 5. 验证策略 -- **训练集 vs 测试集**：ROC是否在测试集上报告？（训练集ROC无意义，必定 inflate） -- **交叉验证**：是否用k-fold CV？折数多少？（k=5或10标准） -- **外部验证**：是否在独立数据集上验证？（金标准） -- **时间验证**：如果是时序数据，是否用未来数据验证？ - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -|------|------|---------| -| 数据泄露 | 测试集信息用于特征选择/模型调参 | 问：特征选择是否在交叉验证循环内？ | -| 阈值报告偏差 | 只报告最优阈值，忽略实际应用阈值 | 看是否说明了阈值选择依据 | -| 不平衡数据用ROC | 正样本<5%时ROC AUC可能虚高 | 问：是否同时报告了PR曲线？ | -| 过拟合 | 训练集AUC=0.99，测试集AUC=0.75 | 比较训练/测试性能差距 | -| 置信区间缺失 | AUC=0.85但无CI，无法判断可靠性 | 看是否报告95% CI或标准误 | - ---- - -## 批判性提问模板 - -1. "AUC是在训练集还是独立测试集上计算的？" -2. "如果数据不平衡，是否同时报告了PR-AUC？" -3. "最优截断点的选择依据是什么？是否符合临床场景？" -4. "置信区间宽度如何？样本量是否足够？" -5. "与现有方法比较时，是否用了DeLong检验？" -6. "验证策略是什么？k-fold CV？外部验证？时间验证？" diff --git a/skills/literature-qa/chart-reading/Western Blot条带图.md b/skills/literature-qa/chart-reading/Western Blot条带图.md deleted file mode 100644 index cc0f3ead..00000000 --- a/skills/literature-qa/chart-reading/Western Blot条带图.md +++ /dev/null @@ -1,76 +0,0 @@ -# Western Blot 条带图 - -## 适用场景 - -Western Blot（免疫印迹）条带的半定量分析，包括磷酸化蛋白检测、总蛋白表达、翻译后修饰（PTM）分析。是生物医学研究中蛋白层面最常用的验证方法。 - ---- - -## 核心阅读维度 - -### 1. 内参（Loading Control）选择与验证 - -| 内参类型 | 常用蛋白举例 | 适用场景 | 风险 | -| ---------------- | ------------------------- | --------------------------------- | --------------------------------- | -| 管家蛋白 | GAPDH、β-Actin、β-Tubulin | 普通全细胞裂解液 | 可能受实验条件调控，不稳定 | -| 总蛋白染色 | Ponceau S、Coomassie | 磷酸化蛋白等 PTM 研究 | 更可靠但定量精度稍低 | -| 上样量控制 | BCA/Braford 定量后等量上样 | 所有WB | 只能控制总蛋白量，不能控制转膜效率 | - -**关键检查点**： -- 管家蛋白是否在所有实验条件下表达稳定？（处理组 vs 对照组） -- 是否验证了内参在线性范围内？（做了稀释曲线） -- 磷酸化蛋白是否用了总蛋白或胞浆/胞核分馏的内参，而非全细胞管家蛋白？ - -### 2. 条带完整性检查 - -- **裁剪规范**：同一膜上是否清晰标注了裁剪分界线？（磷酸化条带和总蛋白条带来自同一膜的不同抗体剥离/重孵育是常规做法，但必须标注） -- **非相邻泳道拼接**：同一膜上不相邻泳道被拼接到一起时，是否有明确分界线/空格标注 -- **曝光均匀性**：整张膜的背景是否均匀？边缘泳道是否与中间泳道曝光一致？ - -### 3. 定量归一化方式 - -``` -归一化值 = (目标条带强度 - 背景) / (内参条带强度 - 背景) -``` - -- **必须确认**：归一化是 Target / Loading Control，还是先归一化到对照组再计算 fold change？ -- **内参一致性**：内参条带强度在所有泳道中是否相近（CV<15%）？如果差异过大，内参本身不可靠 - -### 4. 线性范围与饱和检测 - -- **饱和信号**：条带纯白（像素值=255）= 饱和，数据不可用 -- **线性范围**：信号强度应与上样量成比例；过饱和导致定量偏低 -- **检测方法**：是否做了浓度梯度（dilution series）验证线性范围？ -- **多张膜比较**：如果目标条带和内参来自不同膜，是否有交叉参照（bridging）？ - -### 5. 背景扣除与 ROI 设置 - -- **背景region**：使用膜上无蛋白区域的平均强度，而非纯白背景 -- **ROI一致性**：所有条带使用相同大小的矩形/自由形状选区 -- **局部背景**：每个条带附近扣除各自的局部背景，而非全膜平均背景 - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -| ---------------------- | -------------------------------------------- | -------------------------------------- | -| **内参不稳定** | 管家蛋白在不同处理组间表达差异显著 | 检查内参条带灰度是否在所有泳道中相近 | -| **饱和条带定量** | 曝光过曝导致条带变白，灰度值被截断 | 检查条带是否纯白、是否有光晕 | -| **膜面不均导致假差异** | 转膜时泳道边缘效应造成中间vs边缘信号差异 | 检查同一膜不同区域的信号是否一致 | -| **条带裁剪误导** | 同一膜不同区域拼接到一起，假装是不同条件 | 寻找膜上泳道间距/背景的不自然断裂 | -| **过曝掩盖阴性** | 某处理组信号强到过曝，无法与真正阴性区分 | 检查所有条带是否都在线性范围内 | -| **多重比较未校正** | 3个以上处理组的WB，用独立t检验而不用ANOVA | 检查统计方法部分 | - ---- - -## 关键问题模板 - -1. "内参（管家蛋白）的表达是否真的稳定？有没有独立验证其在实验条件下的稳定性？" -2. "所有条带是否都在相机/底物的线性动态范围内？有没有饱和条带被纳入定量？" -3. "归一化方法是 Target/内参 还是其他？不同膜之间如何校正？" -4. "条带是否来自同一张膜的同一次转印？是否有拼接？是否有非相邻泳道的裁剪？" - -## 一句话总结 - -> WB 条带的核心是**"查内参、防饱和、验线性、看裁剪"**——条带灰度只是数字，背后的内参稳定性、线性范围和膜面均匀性才是定量可信度的根基。 diff --git a/skills/literature-qa/chart-reading/免疫荧光定量图.md b/skills/literature-qa/chart-reading/免疫荧光定量图.md deleted file mode 100644 index 6759b4ed..00000000 --- a/skills/literature-qa/chart-reading/免疫荧光定量图.md +++ /dev/null @@ -1,69 +0,0 @@ -# 免疫荧光定量图 - -## 适用场景 - -免疫荧光（IF）、共聚焦显微镜、定量荧光显微图像分析。常见于蛋白定位、共定位分析、细胞计数、荧光强度定量。是细胞生物学和分子病理学的核心可视化手段。 - ---- - -## 核心阅读维度 - -### 1. 共定位系数（Colocalization Coefficients） - -| 系数名称 | 公式/含义 | 取值范围 | 适用场景 | -| ------------------------- | ------------------------------------------ | -------- | ------------------------------------- | -| **Pearson 相关系数 (rP)** | 协方差标准化，衡量线性相关 | -1 到 +1 | 两通道信号密度相近时最可靠 | -| **Manders M1 / M2** | 通道1/通道2 中共定位像素占总像素的比例 | 0 到 1 | 两通道信号强度差异大时使用 | -| **Manders Overlap (R)** | 实际重叠与最大可能重叠之比 | 0 到 1 | 通用共定位指标 | -| **Li's ICQ** | 基于像素强度相关性的离散度量化 | -0.5 到 +0.5 | 介于 Pearson 和 Manders 之间 | - -**关键解释**： -- M1 = 通道1 中与通道2 共定位的信号 / 通道1 总信号 -- M2 = 通道2 中与通道1 共定位的信号 / 通道2 总信号 -- M1 ≠ M2 时，说明两蛋白的表达量不同但空间重叠 - -### 2. 图像采集质量控制 - -- **Sequential scanning**：双通道或多通道必须用顺序扫描（而非同时激发），以消除串扰（cross-talk） -- **光路校准**：两种荧光通道的发射光谱是否有重叠？是否有 spectral bleed-through？ -- **ROI 选择**：使用 Lasso 工具选择包含荧光信号的区域，排除无荧光背景 -- **阈值设定**：是否使用了自动或手动的强度阈值排除背景噪声？ - -### 3. 荧光强度定量 - -- **平均荧光强度**（Mean Intensity）：总荧光强度 / 细胞数或面积 -- **阳性率**（% Positive）：荧光强度超过阈值的细胞占总细胞数的比例 -- **比值法**：如 p-FOXO3a / Total FOXO3a，需要双通道染色 -- **细胞计数**：DAPI 计数总细胞，特定标记阳性细胞 = 阳性率 - -### 4. 伪彩色与展示规范 - -- 伪彩色（Fire、LUT）是否正确标注了颜色条？ -- 不同通道的对比度/亮度调整是否分别进行？（不能对单通道单独调亮） -- 图像是否经过去噪、反卷积等处理？处理参数是否报告？ - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -| -------------------- | ---------------------------------------------- | ---------------------------------- | -| **串扰未消除** | 两通道同时激发导致光谱重叠，虚假共定位 | 检查是否 sequential scan | -| **阈值主观设定** | 不同图像用不同阈值，导致假阳性/阴性 | 阈值是否在所有图像中一致？是否有自动算法？ | -| **内参不稳定** | 用 GAPDH 或 Actin 做荧光定量内参 | 荧光定量通常不用内参，应报告总蛋白或细胞数 | -| **ROI 选区偏倚** | 只选荧光强的区域，排除弱信号 | 检查是否全视野分析还是选择性分析 | -| **过度处理图像** | 去噪/反卷积过度导致信号失真 | 要求原始未处理图像（raw data） | -| **噪声被当信号** | 弱阳性但未设阈值，背景噪声被计入荧光强度 | 检查信噪比和阈值设定 | - ---- - -## 关键问题模板 - -1. "共定位分析用的是 Pearson 相关系数还是 Manders 系数？两个系数的物理含义分别是什么？" -2. "图像采集是否用了 sequential scanning 以消除通道间串扰？" -3. "荧光强度定量是用了哪种指标（平均强度、峰值强度、阳性率）？如何排除背景噪声？" -4. "是否有光谱重叠（bleed-through）问题导致假性共定位？是否做了单通道对照确认？" - -## 一句话总结 - -> 免疫荧光的核心是**"溯光源、看共定位、防串扰、查阈值"**——荧光图像的美观不等于定量可靠，串扰和阈值是最容易被忽视的两大杀手。 diff --git a/skills/literature-qa/chart-reading/折线图与时间序列.md b/skills/literature-qa/chart-reading/折线图与时间序列.md deleted file mode 100644 index cb39599b..00000000 --- a/skills/literature-qa/chart-reading/折线图与时间序列.md +++ /dev/null @@ -1,84 +0,0 @@ -# 折线图与时间序列 - -## 适用场景 - -展示随时间变化的连续数据，如药物释放曲线、浓度-时间曲线、动力学监测、累积响应。常见于药物递送、组织工程、药代动力学实验。 - ---- - -## 核心阅读维度 - -### 1. 释放模型识别 - -| 模型 | 方程 | 特征 | 适用场景 | -| ------------ | --------------------------------------- | --------------------------------- | ------------------------------- | -| 零级（Zero-order） | Qt = Q0 + K0t | 直线；释放速率与浓度无关 | 渗透泵、控释制剂 | -| 一级（First-order） | Ln(Q0 - Qt) = LnQ0 - K1t | 指数衰减；速率与剩余量成正比 | 水溶性药物从多孔基质 | -| Higuchi | Qt = KH × t^0.5 | 平方根时间关系；扩散控制 | 从矩阵扩散的药物 | -| Korsmeyer-Peppas | Mt/M∞ = K × t^n | 幂律模型；n 决定释放机制 | 聚合物水凝胶、纤维、薄膜 | -| Weibull | Mt/M∞ = 1 - exp[-(t-Ti)/λ]^m | S 型曲线；Ti=延迟时间，λ=尺度参数 | 复杂释放机制 | - -### 2. Korsmeyer-Peppas 释放指数 n 的解释（圆柱形体系） - -| n 值 | 释放机制 | -| ----------------- | ------------------------------- | -| n < 0.45 | Fick 扩散（纯扩散） | -| 0.45 < n < 0.89 | 异常传输（扩散+溶胀协同） | -| n = 0.45 | 菲克扩散（Case I） | -| 0.45 < n < 0.89 | 非菲克扩散（异常传输） | -| n > 0.89 | Case-II 传输（溶胀控制） | - -> 注意：上述 n 值仅对圆柱形几何（如片剂、纤维）有效。不同几何形状的临界值不同。 - -### 3. 突释效应（Burst Release） - -- **定义**：初始阶段药物快速释放，通常在 t=0 附近出现 -- **评估**：突释比例 = (M1 / M∞) × 100%，其中 M1 是第一个时间点的释放量 -- **问题**：突释过大（>30%）可能导致初期血药浓度过高，或导致药物浪费 -- **追问**：是否有突释？突释比例是多少？作者是否解释原因（表面药物、孔道效应）？ - -### 4. 平台期与平衡 - -- **平台期识别**：曲线趋于平坦时，释放达到平衡或耗竭 -- **关键问题**： - - 平台期的释放百分比是多少？（60%？80%？） - - 是否达到 100%？（未达到 = 残留药物可能被困在基质中） - - 平台期持续多久？（长期缓释 vs 短效） - -### 5. 模型拟合质量 - -- **R² 拟合优度**：高 R²（>0.95）不代表模型正确，只能说明拟合紧密 -- **AIC/BIC 模型比较**：不同模型比较时用 AIC 或 BIC，不只用 R² -- **残差分析**：残差是否随机分布？（系统性残差 = 模型错误） -- **外推风险**：不要用模型外推至实验未覆盖的时间范围 - -### 6. 数据呈现完整性 - -- **原始数据点**：是否每个时间点都标注了独立样本（n=?）？ -- **误差棒类型**：SD（离散度）vs SEM（均值精度）vs CI -- **时间点密度**：初期时间点是否足够密集？（突释阶段通常需要更密集的采样） - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -| ------------------ | ------------------------------------------ | ---------------------------------- | -| **模型选择偏倚** | 用 R²最高的模型，但该模型物理意义不符 | 检查作者是否报告了多个模型及 AIC/BIC | -| **突释未量化** | 忽略初始快速释放阶段，只报告稳态释放 | 看 t=0 或第一个时间点的释放比例 | -| **假零级** | 曲线看似直线但实际是一级+扩散的叠加 | 检查不同浓度起始点的释放曲线是否平行 | -| **时间范围不充分** | 采样时间不足以判断是否达到平台期 | 观察曲线末端是否明显趋于平坦 | -| **误差棒类型混淆** | SEM 被标注为 SD，制造"精确"假象 | 方法或图注中是否注明误差棒类型 | - ---- - -## 关键问题模板 - -1. "释放曲线符合哪种模型？n（Korsmeyer-Peppas 指数）或 K（ Higuchi 常数）的物理意义是什么？" -2. "是否存在突释效应？突释比例是多少？是否会影响临床疗效或安全性？" -3. "释放是否达到平台期？残余药物比例是多少？这对体内药效持续时间有什么暗示？" -4. "模型拟合的 R² 是否只是表面好看——残差是否有系统性偏差？换用其他模型是否更合理？" - -## 一句话总结 - -> 折线图的核心是**"辨模型、看突释、查平台、验残差"**——药物释放不是看曲线"漂亮"，而是看机制是否自洽、平台是否有意义。 diff --git a/skills/literature-qa/chart-reading/散点图与气泡图.md b/skills/literature-qa/chart-reading/散点图与气泡图.md deleted file mode 100644 index eedd2c40..00000000 --- a/skills/literature-qa/chart-reading/散点图与气泡图.md +++ /dev/null @@ -1,87 +0,0 @@ -# 散点图与气泡图 - -## 适用场景 - -展示两个连续变量之间的关系，或增加第三个维度（气泡大小）展示多变量关联。常见于相关性分析、聚类结果可视化、剂量-反应关系。 - -## 读图维度 - -### 1. 坐标轴与变量定义 - -- **X/Y轴分别代表什么变量？** - - 是否为连续变量？单位是什么？ - - 是否存在对数转换（如 log10、log2）？ - - 轴范围是否被截断（truncated axis）？ -- **变量关系类型**： - - 自变量→因变量（实验设计） - - 双向关联（观察性研究） - - 时间序列（X轴为时间） - -### 2. 数据点分布特征 - -- **整体趋势**： - - 线性 / 非线性 / 无明显趋势 - - 正相关 / 负相关 / U型 / 倒U型 -- **离散程度**： - - 点的密集区域在哪里？ - - 是否存在明显分层（如病例 vs 对照）？ -- **异常值**： - - 远离主群体的点是否被标注？ - - 异常值是真实数据还是测量误差？ - -### 3. 分组与着色 - -- **颜色/形状编码**： - - 不同颜色代表什么分组？（疾病亚型、处理组、时间窗） - - 是否使用渐变色表示连续变量？ -- **分组间差异**： - - 各组是否呈现不同的分布模式？ - - 是否存在组间重叠？ - -### 4. 气泡图专属维度 - -- **气泡大小代表什么？** - - 样本量？统计显著性？效应量？ - - 大小比例是否线性？ -- **三维信息整合**： - - X/Y/Size 三个维度是否讲述一致的故事？ - - 是否存在大样本量但效应小的" dominant "气泡？ - -### 5. 辅助元素 - -- **回归线/趋势线**： - - 是否添加了拟合线？什么模型？（线性、LOESS、多项式） - - 置信区间（阴影区域）是否显示？ -- **参考线**： - - 是否有阈值线（如 cutoff、基线）？ - - 象限划分线（如 mean±SD）？ - -### 6. 统计标注 - -- **相关系数**： - - Pearson r / Spearman ρ 值是多少？ - - p-value 或 q-value 是否标注？ -- **样本量**： - - n = ? 是否每组的 n 都足够？ - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **过度绘制** (Overplotting) | 点过多重叠成"实心团" | 掩盖真实分布，看不出密度差异 | -| **假相关** | X和Y实际受第三方变量Z驱动 | 将伴随关系误读为因果关系 | -| **轴截断误导** | Y轴不从0开始 | 放大微弱差异，制造虚假趋势 | -| **气泡大小误导** | 面积 vs 半径编码混淆 | 读者对大小的感知被扭曲 | -| **选择性标注** | 只标注"有趣"的点 | 确认偏误，忽略不支持假设的数据 | -| **伪Replication** | 一个患者多个时间点作为独立点 | 违反独立性假设，虚增样本量 | - -## 关键问题模板 - -1. "散点图显示 X 和 Y 的相关系数为 r=0.XX，但散点分布呈漏斗形/曲线形，线性 Pearson 相关是否合适？" -2. "图中标注的离群点是否被作者排除？排除标准是什么？是否影响结论？" -3. "气泡大小代表样本量，最大的气泡是否主导了视觉印象，而小样本研究的结果可能更不稳定？" -4. "如果按第三个变量分层（如性别、年龄组），相关性是否仍然成立？" - -## 一句话总结 - -> 散点图的核心是**"看趋势、看离散、看分组、防过绘"**——确认变量关系形态，警惕视觉假象和伪相关。 diff --git a/skills/literature-qa/chart-reading/显微照片与SEM图.md b/skills/literature-qa/chart-reading/显微照片与SEM图.md deleted file mode 100644 index 98a0757e..00000000 --- a/skills/literature-qa/chart-reading/显微照片与SEM图.md +++ /dev/null @@ -1,81 +0,0 @@ -# SEM / TEM 图像 - -## 适用场景 - -扫描电子显微镜（SEM）和透射电子显微镜（TEM）获取的材料形貌、纳米结构、孔径分布、纤维形态等图像。常见于生物材料、组织工程支架、纳米药物递送、植入物表面改性研究。 - ---- - -## 核心阅读维度 - -### 1. 标尺（Scale Bar）与放大倍数 - -- **标尺必须存在**：所有publication-quality的显微图像必须包含标尺 -- **标尺长度选择**：标尺长度应约为图像宽度的 5-10%，过小难以辨认，过大遮挡视野 -- **放大倍数标注**：通常以 "scale bar = X μm" 或 "mag = 50,000×" 形式标注 -- **实际尺寸计算**：用标尺测量而非依赖仪器显示的放大倍数（仪器显示的放大倍数可能有5-10%误差） - -**SEM 放大倍数误差**：SEM 放大倍数通常有 5-10% 的系统误差，ISO 16700 标准规定了校准方法 -**TEM 放大倍数误差**：TEM 放大倍数误差可达 5-10%，需用衍射光栅或晶格标准品校准 - -### 2. 代表性图像原则 - -- **多视野原则**：是否展示了同一样本的多个视野（至少 3 个）？ -- **统计报告**：图像数量 n=？（如 "n=5 independent images"） -- **正负对照**：是否有代表性图像和阴性/异常图像的并列展示？ - -### 3. 图像处理与增强 - -| 处理类型 | 可接受场景 | 不可接受场景 | -| ------------------- | ----------------------------------- | ----------------------------------- | -| 亮度/对比度调整 | 全图统一调整 | 只调整局部区域 | -| 伪彩色 | 假色SEM（用于高亮特定结构） | 掩盖真实对比度差异 | -| 图像拼接 | 明确标注拼接线 | 隐藏拼接痕迹 | -| 锐化/降噪 | 降噪不损失真实结构细节 | 锐化过度产生人工边缘 | - -### 4. SEM 特有检查项 - -- **荷电效应（Charging）**：图像中白色条纹/区域 = 荷电，信号失真 -- **焦深（DOF）**：高倍SEM焦深浅，图像是否清晰取决于是否在焦平面 -- **倾斜角度**：如果样本经过倾斜处理，实际形貌可能被扭曲 -- **加速电压**：5-20 kV 影响穿透深度和信号类型（SE vs BSE） - -### 5. TEM 特有检查项 - -- **电子衍射标定**：晶格条纹是否与标准数据库匹配？ -- **样品厚度**：过厚样品导致对比度下降、晶格模糊 -- **辐照损伤**：生物样品在高电子剂量下可能受损（尤其 cryo-TEM） -- **光阑选择**：明场 vs 暗场，是否正确使用 - -### 6. 定量分析规范 - -- **测量工具**：ImageJ / Fiji 是标准工具，必须报告软件版本 -- **样本量**：至少测量 200 个颗粒/结构（ISO 13322-1:2004） -- **统计报告**：报告均值 ± SD / 中位数（IQR），并说明测量的结构数量 n -- **校准验证**：测量前是否用标准品（如 NIST SRM 8820）对仪器进行了校准验证 - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -| ------------------ | ------------------------------------------ | ---------------------------------- | -| **标尺不准确** | 仪器显示放大倍数与实际不符 | 用标尺实际测量并与标注尺寸对比 | -| **单视野代表全貌** | 只展示一个最"好看"的图像 | 是否有 n≥3 的多视野重复 | -| **荷电当成结构** | 白色条纹区域是荷电假象，不是真实样品结构 | 检查图像背景是否均匀 | -| **图像过度处理** | 对比度/锐化过度，引入人工结构 | 要求原始未处理图像（TIFF） | -| **TEM 样品过厚** | 过厚样品导致晶格模糊、对比度差 | 图像整体发暗、晶格条纹不清晰 | -| **尺寸测量不报告n**| 只说"粒径约 50nm"但不报告测量了多少颗粒 | 方法中是否说明了统计的 n | - ---- - -## 关键问题模板 - -1. "标尺的实际长度是多少？是否与标注一致？仪器校准是否使用了 NIST 可追溯标准（如 SRM 8820）？" -2. "每张图像来自几个独立视野（n=？）？是否有多个样本/制备条件的重复？" -3. "图像是否经过处理（如对比度调整、伪彩色）？处理是否全图统一，还是只处理了局部？" -4. "SEM 图像中是否存在荷电效应（白色条纹/区域）？TEM 图像中样品厚度是否足以获得清晰晶格？" - -## 一句话总结 - -> 显微图像的核心是**"验标尺、查多野、审处理、核校准"**——漂亮的图像不等于可靠的数据，标尺准确性、样本代表性和仪器校准是三个生命线。 diff --git a/skills/literature-qa/chart-reading/条形图与误差棒.md b/skills/literature-qa/chart-reading/条形图与误差棒.md deleted file mode 100644 index 1bddb6c1..00000000 --- a/skills/literature-qa/chart-reading/条形图与误差棒.md +++ /dev/null @@ -1,81 +0,0 @@ -# 条形图与误差棒 - -## 适用场景 - -比较不同组别在某个指标上的均值/频数差异，带误差棒表示不确定性。常见于组间比较、时间趋势、分类变量分布。 - -## 读图维度 - -### 1. 条形含义与基准 - -- **条形高度代表什么？** - - 均值？中位数？频数？百分比？ - - 如果是均值，原始数据分布是否对称？ -- **Y轴起点**： - - 是否从0开始？（条形图必须从0开始！） - - 如果截断，差异被放大了多少倍？ - -### 2. 误差棒解读 - -- **误差棒类型**： - - **SD**（标准差）：数据离散程度 - - **SEM**（标准误）：均值估计精度 = SD/√n - - **95% CI**（置信区间）：统计推断范围 - - **IQR**（四分位距）：非参数离散度 -- **误差棒重叠 ≠ 无显著差异** - - SEM 重叠不代表 p > 0.05 - - 必须看正式的统计检验结果 -- **误差棒长度暗示样本量** - - 相同 SD 下，n 越大 SEM 越小 - - 极小的误差棒可能暗示 n 很大或 SD 被低估 - -### 3. 分组结构 - -- **X轴分组逻辑**： - - 单因素分组？双因素交叉？ - - 组间是否存在自然顺序（如时间、剂量）？ -- **对照组设置**： - - 是否有合适的对照/基线？ - - 对照组是否为"健康人"还是"安慰剂"？ - -### 4. 统计显著性标注 - -- **星号/字母系统**： - - * p<0.05, ** p<0.01, *** p<0.001 - - 不同字母代表组间差异显著（如 a vs b） -- **多重比较校正**： - - 是否进行了 Bonferroni / FDR 校正？ - - 未校正的多重 t 检验 inflate Type I error - -### 5. 条形图变体识别 - -- **堆叠条形图 (Stacked)**： - - 各部分之和是否有意义（如 100%）？ - - 比较不同组的同一子类别是否困难？ -- **分组条形图 (Grouped)**： - - 每组内各条的顺序是否一致？ - - 视觉负担是否过大（>5组×5条件）？ -- **水平条形图**： - - 标签过长时的解决方案 - - 阅读方向改变不影响解读逻辑 - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **Y轴不从0开始** | 检查Y轴最小值 | 微小差异被放大数倍，误导读者 | -| **SEM冒充SD** | 看误差棒图例标注 | SEM 总是更小，制造"精确"假象 | -| **双向条形图误导** | 正负方向同时延伸 | 读者对绝对值的比较被干扰 | -| **3D条形图** | 立体效果 | 深度扭曲高度感知，毫无意义 | -| **未校正的多重比较** | 大量星号但没有校正说明 | 假阳性率飙升 | - -## 关键问题模板 - -1. "误差棒标注为 mean±SEM，但样本量 n=5，用 SEM 是否故意缩小误差视觉效果？" -2. "条形图显示 A 组比 B 组高 30%，但 Y 轴从 50 开始而非 0，实际差异是否被夸大？" -3. "进行了 10 组两两比较，但 p 值未校正，标注的显著性是否可信？" -4. "堆叠条形图中，上层类别的变化是否被下层基线变化所掩盖？" - -## 一句话总结 - -> 条形图的核心是**"看基准、看误差、看校正、防截断"**——误差棒类型决定解读方式，Y轴起点决定差异真实度。 diff --git a/skills/literature-qa/chart-reading/桑基图与弦图.md b/skills/literature-qa/chart-reading/桑基图与弦图.md deleted file mode 100644 index 7ac58793..00000000 --- a/skills/literature-qa/chart-reading/桑基图与弦图.md +++ /dev/null @@ -1,88 +0,0 @@ -# 桑基图与弦图 - -## 适用场景 - -**桑基图 (Sankey Diagram)**：展示流量/数量的流向和分配，常见于患者诊疗路径、多组学数据整合、能量/物质流动、临床试验病例流转。 - -**弦图 (Chord Diagram)**：展示双向关系或配对关联的密度，常见于细胞互作、基因共表达模块间的共享基因、不同亚群间的迁移/转换关系。 - -## 读图维度 - -### 桑基图 - -#### 1. 节点与层级 - -- **节点代表什么？** - - 分类状态（如疾病分期、治疗阶段、分子亚型） - - 时间节点（如基线→3月→6月→12月） - - 数据类型（如基因组→转录组→蛋白组） -- **层级逻辑**： - - 从左到右是否有自然的时序或因果顺序？ - - 是否可以逆流？（如治疗后复发回到前期状态） - -#### 2. 流量与宽度 - -- **线条宽度**： - - 与流量/数量成正比 - - 视觉上线条宽度是否可准确比较？ -- **流量守恒**： - - 进入节点的总流量是否等于流出总流量？ - - 如有损失（如失访、死亡），是否标注了"流失"路径？ - -#### 3. 路径追踪 - -- **主要路径**： - - 最粗的线代表什么主流向？ - - 是否存在多条并行的重要路径？ -- **罕见路径**： - - 极细的线条是否代表真实的小比例事件？ - - 还是被视觉压缩到几乎看不见？ - -### 弦图 - -#### 1. 环形节点 - -- **圆周上的节点**： - - 代表什么实体？（细胞类型、基因模块、样本组） - - 节点弧长是否代表大小？（如细胞数量、模块基因数） -- **颜色编码**： - - 节点颜色是否代表分组/类别？ - - 弦的颜色是源节点色、目标节点色、还是混合色？ - -#### 2. 弦的解读 - -- **弦的粗细**： - - 代表两节点间关系强度（如配体-受体对数、共享基因数） - - 弦越粗 = 相互作用越强 -- **弦的方向**： - - 是否有方向性？（如从细胞A到细胞B的分泌关系） - - 还是纯无向关联？ - -#### 3. 矩阵关系 - -- **邻接矩阵 vs 弦图**： - - 弦图是邻接矩阵的可视化 - - 如果弦图过于复杂，看原始矩阵是否更清晰？ -- **自环 (Self-loop)**： - - 节点连接到自身的弦代表什么？（自分泌、模块内调控） - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **桑基图层级过多** | >4 层 | 线条交叉严重，无法追踪路径 | -| **弦图节点过多** | >20 个节点 | 视觉混乱，弦完全重叠 | -| **桑基流量未标注比例** | 只有绝对数 | 无法判断占比，需要同时看百分比 | -| **弦图过度解读方向** | 无方向数据强行加箭头 | 误导为因果关系 | -| **桑基隐藏关键流失** | 未显示失访/死亡分支 | 幸存者偏误，夸大疗效 | - -## 关键问题模板 - -1. "桑基图显示从诊断到治疗有 30% 的病例"流失"，这些病例的临床特征是否与继续治疗的病例不同？是否存在选择偏倚？" -2. "弦图显示细胞类型 A 和 B 之间有大量互作，但这是基于单一数据集的预测，是否有独立的细胞实验验证？" -3. "桑基图的最粗路径是否代表了作者希望强调的"理想路径"，而较细的路径实际上也很重要但被视觉弱化了？" -4. "弦图中模块 1 和模块 2 的共享基因有 50 个，但两个模块各自的基因数分别是 200 和 300，Jaccard 重叠度其实只有 10%，弦的粗细是否放大了关联感？" - -## 一句话总结 - -> 桑基图的核心是**"追路径、看守恒、防流失"**，弦图的核心是**"看强度、防混乱、验证方向性"**——流量可视化的真实度取决于未显示部分的透明度。 diff --git a/skills/literature-qa/chart-reading/森林图与Meta分析.md b/skills/literature-qa/chart-reading/森林图与Meta分析.md deleted file mode 100644 index 02d3b754..00000000 --- a/skills/literature-qa/chart-reading/森林图与Meta分析.md +++ /dev/null @@ -1,81 +0,0 @@ -# 森林图与 Meta 分析 - -## 适用场景 - -展示多个独立研究（或亚组分析）的效应量估计及其置信区间，底部汇总合并效应量。是 Meta 分析的标准可视化，也用于亚组分析、敏感性分析、多变量回归结果展示。 - -## 读图维度 - -### 1. 研究/亚组排列 - -- **左侧列标签**： - - 每个方块代表什么？（单个研究、亚组、或总体合并） - - 研究排列顺序：按发表时间？按效应量大小？按权重？ -- **研究特征**： - - 每个研究的样本量（n）？ - - 研究设计类型（RCT vs 观察性）？ - - 随访时间、干预剂量等关键特征？ - -### 2. 效应量与置信区间 - -- **效应量指标**： - - HR（风险比）、OR（比值比）、RR（相对风险）、MD（均值差）、SMD（标准化均值差） - - 效应量 = 1（或 0）线代表什么？（无效应线） -- **置信区间 (CI)**： - - 95% CI 是否跨越无效线？→ 该研究/亚组不显著 - - CI 宽度反映精度：宽 = 样本小/异质性大；窄 = 样本大/精度高 -- **菱形（汇总估计）**： - - 中心点 = 合并效应量 - - 宽度 = 95% CI - - 菱形是否跨越无效线？→ 总体是否显著 - -### 3. 权重分配 - -- **方块大小**： - - 通常与权重（Weight %）成正比 - - 大样本研究 = 大方块 = 对合并结果影响大 -- **权重合理性**： - - 固定效应模型 vs 随机效应模型权重不同 - - 是否有个别研究权重过高（>50%）？ - -### 4. 异质性评估 - -- **I² 统计量**： - - 0-25%：低异质性；25-50%：中等；50-75%：高；>75%：很高 - - I² 高 → 研究间差异大，合并需谨慎 -- **Q 检验 (Cochran's Q)**： - - p < 0.05 表示存在显著异质性 -- **Tau²**： - - 随机效应模型中的异质性方差 -- **异质性可视化**： - - CI 是否广泛重叠？（重叠多 = 异质性低） - - 是否存在方向相反的研究？ - -### 5. 发表偏倚线索 - -- **图形不对称性**： - - 小样本阴性结果是否缺失？ - - 大样本研究是否聚集在真实效应附近？ -- **Egger 检验 / 漏斗图**： - - 森林图本身不能检测发表偏倚，需配合漏斗图 - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **忽略异质性** | I² > 50% 仍做合并 | 合并结果无意义，亚组差异被掩盖 | -| **固定效应误用** | 高异质性用固定效应 | 低估标准误，假阳性 | -| **亚组分析过多** | 大量亚组，每组仅2-3个研究 | 假阳性，交互作用虚假 | -| **生态学谬误** | 用研究层面特征解释个体层面 | 因果推断层次混乱 | -| **合并不可比研究** | 不同设计、不同人群、不同干预 | Garbage in, garbage out | - -## 关键问题模板 - -1. "森林图显示 I² = 78%，作者仍报告合并 HR=0.85 (95%CI 0.75-0.96)，高异质性下这个合并值是否还有临床意义？" -2. "最大权重的研究贡献了 60% 的权重，如果排除该研究，合并结果是否逆转？" -3. "亚组分析显示男性获益 (HR 0.7) 而女性不获益 (HR 1.0)，交互检验 p=0.03 还是 p=0.2？差异是否真实？" -4. "所有小样本研究的 CI 都跨 1，只有最大样本的研究显示显著，是否存在小样本偏倚或发表偏倚？" - -## 一句话总结 - -> 森林图的核心是**"看方块、看CI、看异质、看权重"**——单个研究的精确度和总体异质性比合并点估计更重要。 diff --git a/skills/literature-qa/chart-reading/火山图与曼哈顿图.md b/skills/literature-qa/chart-reading/火山图与曼哈顿图.md deleted file mode 100644 index 93ccfe33..00000000 --- a/skills/literature-qa/chart-reading/火山图与曼哈顿图.md +++ /dev/null @@ -1,62 +0,0 @@ -# 火山图与曼哈顿图阅读指南 - -## 适用场景 -- 展示大规模差异分析结果（如RNA-seq、GWAS、蛋白质组学） -- 同时展示统计显著性和效应量 -- 识别top hits（关键基因/SNP/蛋白质） - ---- - -## 核心阅读维度 - -### 1. 阈值线 -- **显著性阈值**：通常是水平线（如 p=0.05, p=0.01 或 FDR q=0.05） - - 注意：如果用了-log10(p)，阈值线是水平的；如果用原始p值，是垂直的 - - **多重检验校正**：如果是组学数据（测了上万个基因），必须用FDR（Benjamini-Hochberg）或Bonferroni校正，不能用raw p-value -- **效应量阈值**：通常是垂直线（如 |log2FC| > 1 或 > 0.5） - - 小效应量（|log2FC| < 0.5）即使统计显著，生物学意义可能有限 - -### 2. 象限分布 -- **左上/右上象限**：高显著性 + 大效应量 = 最有生物学意义的hits -- **中间区域**：不显著或效应量小 = 可忽略 -- **对称性**：上调和下调的数量是否对称？不对称可能提示技术偏倚（如文库制备偏差） - -### 3. Top Hits -- **标注的基因/位点**：作者特别标注的点是否有生物学合理性？ -- **已知标记**：是否与先验知识一致？（如已知癌症基因在肿瘤vs正常中差异表达） -- **新颖性**：新发现的hits是否有功能注释支持？ - -### 4. 整体模式 -- **漏斗形/三角形**：理想情况，大效应量对应高显著性 -- **离散分布**：无明确模式 = 可能缺乏系统性差异 -- **批次效应信号**：如果看到按批次聚类的显著点（如所有batch1样本的某基因都高表达）= 技术artifact - -### 5. 曼哈顿图特有（GWAS） -- **基因组位置**：x轴是染色体位置，y轴是-log10(p) -- **峰（Peak）**：连续多个显著SNP = 关联区域（LD block） -- **基因注释**：峰附近是否有已知基因？功能是什么？ -- **Bonferroni阈值**：通常为 5×10^-8（全基因组显著性） -- ** suggestive 阈值**：通常为 1×10^-5（提示性关联，需验证） - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -|------|------|---------| -| 未校正p值 | 用raw p-value而非FDR，假阳性率高 | 看方法部分：是否提到FDR/BH校正？ | -| 效应量过小 | |log2FC|<0.2却标注为"显著差异" | 看垂直阈值线位置 | -| 批次效应 | 技术批次驱动差异而非生物学 | 看是否有按批次聚类的显著点 | -| 样本量不足 | 小样本导致效应量估计不稳定 | 看误差线（如果有）或 replicate 数量 | -| 选择性标注 | 只标注支持结论的点，忽略矛盾点 | 看是否标注了所有已知关键基因？ | - ---- - -## 批判性提问模板 - -1. "多重检验校正方法是什么？如果是组学数据，是否用了FDR？" -2. "效应量阈值是多少？|log2FC|>1还是>0.5？生物学意义如何？" -3. "显著性阈值是raw p-value还是校正后的q-value？" -4. "Top hits是否有先验知识支持？新发现的hits是否有功能注释？" -5. "上调和下调基因数量是否对称？如果不对称，原因是什么？" -6. "如果是曼哈顿图，峰宽是多少？是否跨越多个基因？" diff --git a/skills/literature-qa/chart-reading/热图与聚类图.md b/skills/literature-qa/chart-reading/热图与聚类图.md deleted file mode 100644 index 8cf4fbd2..00000000 --- a/skills/literature-qa/chart-reading/热图与聚类图.md +++ /dev/null @@ -1,65 +0,0 @@ -# 热图与聚类图阅读指南 - -## 适用场景 -- 展示矩阵数据（基因表达、蛋白质丰度、相关性等） -- 识别样本或特征的相似性模式 -- 展示通路/基因集的活性 - ---- - -## 核心阅读维度 - -### 1. 聚类结构（如果存在） -- **行/列聚类**：样本或基因是否形成明显分组？ -- **聚类算法**：通常用层次聚类（hierarchical），距离度量（Euclidean/Pearson/correlation）和链接方法（complete/average/Ward）会影响结果 -- **稳定性**：如果改变距离度量，聚类模式是否仍然稳定？（不稳定的聚类不可靠） - -### 2. 颜色与标准化 -- **颜色梯度**： - - 红-蓝：通常表示上调-下调（表达量） - - 黄-黑-蓝：通常表示激活-中性-抑制（通路活性） - - **必须确认**：caption是否明确说明颜色含义？ -- **标准化方法**： - - **Z-score**：(x - mean) / SD，突出相对变化，适合跨样本比较 - - **Log转换**：用于偏态分布（如RNA-seq count数据） - - **Percentile**：将数据映射到0-100%，丢失绝对量信息 - - **Min-Max**：映射到[0,1]，对异常值敏感 - - **关键问题**：标准化方法是否适合数据类型？（如RNA-seq应该用log2 TPM/FPKM，而不是原始count） - -### 3. 注释条（Annotation Bars） -- **顶部/侧边色条**：是否标注了临床特征（如亚型、分期、治疗响应）？ -- **一致性**：热图聚类模式与注释特征是否一致？（如：热图分3组，注释条也正好分3组 = 好） -- **遗漏信息**：是否有重要协变量未标注？（如批次效应、性别、年龄） - -### 4. 关键区域 -- **高亮块**：作者是否在文中特别指出了某些区域？ -- **边界清晰度**：分组之间的边界是清晰还是模糊？（模糊 = 亚型定义不稳健） -- **一致性**：同一组内的样本是否表现一致？（组内方差大 = 亚型异质性高） - -### 5. 行列选择 -- **样本选择**：是否排除了某些样本？原因是什么？（如QC失败） -- **特征选择**： - - 是全部基因还是筛选后的子集？ - - 筛选标准是什么？（方差top N？差异表达？已知标记？） - - **风险**：先筛选再聚类会造成"循环论证"（看起来有结构，其实是因为你选了有结构的基因） - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -|------|------|---------| -| 循环论证 | 用差异基因做聚类，然后说"我们发现两个亚型" | 看方法部分：聚类用的基因是否预先筛选过？ | -| 批次效应伪装 | 聚类按批次而非生物学分组 | 问：如果有批次注释条，是否与聚类一致？ | -| 颜色误导 | 非对称颜色映射或断裂色阶 | 看color bar是否连续、是否以0为中心 | -| 过度解读 | 把噪声当成模式 | 问：如果随机打乱数据，聚类是否还明显？ | - ---- - -## 批判性提问模板 - -1. "聚类用的基因是如何筛选的？是否预先基于差异表达筛选？" -2. "颜色标准化方法是什么？Z-score、log、还是percentile？" -3. "如果有批次效应，热图是否能区分批次和真实生物学差异？" -4. "聚类结果的稳定性如何？换用不同距离度量或聚类算法，模式是否一致？" -5. "作者声称的亚型数量（如k=3）是否有统计支持？（如gap statistic, silhouette score）" diff --git a/skills/literature-qa/chart-reading/生存曲线.md b/skills/literature-qa/chart-reading/生存曲线.md deleted file mode 100644 index ba8ad37a..00000000 --- a/skills/literature-qa/chart-reading/生存曲线.md +++ /dev/null @@ -1,75 +0,0 @@ -# 生存曲线阅读指南 - -## 适用场景 -- 时间-事件数据（time-to-event） -- 比较不同组别的生存/复发/进展时间 -- 评估预后因素的独立效应 - ---- - -## 核心阅读维度 - -### 1. 曲线形态 -- **中位生存时间（MST）**：曲线下降到50%时对应的时间。 - - 如果曲线始终>50% = 中位未达到，应报告"中位生存期未达到" - - 注意：MST是样本统计量，不是个体预测 -- **早期/晚期分离**： - - 早期分离（曲线在前6个月就分开）= 治疗/因素有快速效应 - - 晚期分离 = 效应随时间累积 - - 始终不分离 = 组间无差异 -- **平台期**：曲线在某段时间水平 = 事件风险暂时消失（如治愈） - -### 2. 删失数据（Censoring） -- **删失标记**：曲线上通常有小竖线表示删失 -- **删失比例**：如果>50%样本删失，估计不可靠（随访时间太短或失访太多） -- **信息删失 vs 随机删失**： - - 信息删失（因副作用退出）= 偏差风险 - - 随机删失 = 可接受 -- ** competing risks **：如果有竞争风险（如癌症死亡 vs 其他原因死亡），标准Kaplan-Meier会高估累积发生率 - -### 3. 统计显著性 -- **Log-rank检验**：比较两条/多条曲线的整体差异。 - - p<0.05 = 组间生存分布有差异 - - 注意：log-rank检验对晚期差异敏感，对早期差异不敏感 -- **Hazard Ratio（HR）**： - - HR>1 = 暴露组风险高（如死亡更快） - - HR<1 = 暴露组风险低（如死亡更慢） - - HR=1 = 无差异 - - **必须看95% CI**：如果CI包含1，即使HR≠1也不显著 -- **Breslow检验 / Tarone-Ware**：如果对早期差异更感兴趣，log-rank可能不够敏感 - -### 4. 样本量与统计效力 -- **每组事件数（number of events）**： - - 总事件数 < 10×变量数 = Cox回归过拟合风险 - - 例如：10个协变量，需要至少100个事件 -- **最小随访时间**：是否足够长？（如5年生存率，但中位随访仅2年 = 不可信） - -### 5. Cox回归假设 -- **比例风险假设（PH assumption）**：HR在不同时间点是否恒定？ - - 检验方法：Schoenfeld残差、log-log图 - - 如果违反：需要分层Cox、时变系数或加速失效模型（AFT） -- **线性假设**：连续协变量的对数风险是否线性？ - - 可用martingale残差检验 - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -|------|------|---------| -| immortal time bias | 从治疗后开始算生存，但治疗前有"免疫"时间 | 看时间0的定义是否合理 | -| lead-time bias | 筛查提前发现疾病，造成"生存期延长"假象 | 问：是否报告了 lead-time 校正？ | -| overfitting in Cox | 变量数 > 事件数/10 | 看事件数和变量数的比例 | -| 忽略竞争风险 | 竞争事件存在时，KM曲线高估累积发生率 | 问：是否用了Fine-Gray竞争风险模型？ | -| 中位生存未标注 | 曲线始终>50%，但作者说"中位生存X月" | 这是不可能的，应质疑 | - ---- - -## 批判性提问模板 - -1. "中位随访时间是多少？是否足够观察主要终点？" -2. "删失比例是多少？删失原因是否随机？" -3. "HR的95% CI是否包含1？如果包含，差异不显著" -4. "是否检验了比例风险假设？如果违反，用了什么替代方法？" -5. "如果存在竞争风险，是否用了适当的统计方法？" -6. "Cox回归的变量数/事件数比例是否合理？" diff --git a/skills/literature-qa/chart-reading/箱式图与小提琴图.md b/skills/literature-qa/chart-reading/箱式图与小提琴图.md deleted file mode 100644 index 1e2d4205..00000000 --- a/skills/literature-qa/chart-reading/箱式图与小提琴图.md +++ /dev/null @@ -1,58 +0,0 @@ -# 箱式图与小提琴图阅读指南 - -## 适用场景 -- 展示连续变量在不同分组间的分布 -- 比较多组间的中位数/均值差异 -- 识别异常值和数据质量 - ---- - -## 核心阅读维度 - -### 1. 集中趋势 -- **中位数线位置**：组间中位数差异是否明显？ -- **均值标记**：如果标注了均值（通常用菱形或叉号），与中位数是否接近？（接近=对称分布，远离=偏态） - -### 2. 离散程度 -- **IQR（四分位距）**：箱体高度代表50%数据的范围。IQR大 = 组内异质性高 -- ** whisker 长度**：通常延伸至 1.5×IQR 或最值。Whisker长 = 分布尾部重 -- **小提琴图宽度**：某值处越宽 = 该值附近数据密度越高 - -### 3. 异常值 -- **Outlier 标记**：通常是箱外的点。 - - 判断：是生物学真实差异还是技术错误？ - - 样本量小（n<10）时，每个点都很重要，不要轻易忽略 -- **样本量影响**：n<5时箱式图不可靠，应该看原始散点 - -### 4. 统计显著性 -- **p值标注**：* p<0.05, ** p<0.01, *** p<0.001 -- **检验方法**： - - 两组：t-test（正态）或 Mann-Whitney U（非正态） - - 多组：ANOVA（正态）或 Kruskal-Wallis（非正态） - - 如果数据明显偏态，用参数检验（t/ANOVA）是**错误**的 -- **多重比较校正**：如果做多次两两比较，是否用了 Bonferroni / FDR 校正？ - -### 5. 分组与对照 -- **对照组设置**：是否有适当的对照？（如正常组织 vs 病变组织） -- **组间样本量平衡**：如果一组 n=50 另一组 n=5，差异可能不可靠 -- **配对设计**：如果是配对样本（如治疗前后），应该用配对检验，而非独立检验 - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -|------|------|---------| -| Y轴截断 | 截断Y轴放大微小差异 | 检查Y轴是否从0开始（比值类）或是否有断裂标记 | -| 不标n值 | 无法判断统计效力 | 看caption或方法部分是否有样本量说明 | -| 用均值±SEM代替箱式图 | SEM小，掩盖真实离散度 | SEM = SD/√n，永远比SD小，是"美化"手段 | -| 忽略正态性假设 | t-test用于极端偏态数据 | 看是否报告了Shapiro-Wilk检验或使用非参数检验 | - ---- - -## 批判性提问模板 - -1. "组间差异的效应量（effect size）是多少？ Cohen's d > 0.5 才算中等效应" -2. "异常值是否影响了中位数？ 去掉异常值后结论是否改变？" -3. "如果改用非参数检验，p值是否仍然显著？" -4. "样本量是否足够检测到这个效应量？"（可用G*Power反推） diff --git a/skills/literature-qa/chart-reading/组织学半定量图.md b/skills/literature-qa/chart-reading/组织学半定量图.md deleted file mode 100644 index 356b1423..00000000 --- a/skills/literature-qa/chart-reading/组织学半定量图.md +++ /dev/null @@ -1,80 +0,0 @@ -# 组织学半定量图 - -## 适用场景 - -关节软骨、骨、肌肉、皮肤等组织的染色评估（H&E、Safranin O、Masson trichrome、番红O-固绿等），以及基于评分系统的半定量组织学分析。常见于骨科、再生医学、创伤修复研究。 - ---- - -## 核心阅读维度 - -### 1. 评分系统识别与适用范围 - -| 评分系统 | 分项数 | 满分 | 适用场景 | 特点 | -| -------------- | ------ | ------ | ----------------------------- | --------------------------------- | -| **O'Driscoll** | 8 | 24 | 动物软骨修复 | 评估结构、细胞、基质、矿化等 | -| **Modified O'Driscoll** | 8 | 24+ | 软骨修复 | 加入hyaline cartilage比例 | -| **ICRS II** | 14 | 每项100mm VAS | 人/动物软骨 | 14个独立评分项，可视化模拟评分尺 | -| **Mankin** | 4-5 | 14 | 骨关节炎分级 | 结构、细胞、Safranin O、潮线 | -| **OARSI** | 多维度 | 层级制 | 人膝关节骨关节炎组织学 | 更适合退行性病变 | -| **Pineda** | 4 | 12 | 软骨修复 | 简单快速 | - -### 2. 评分者间一致性（Inter-rater Reliability） - -- **评估方法**：组内相关系数（ICC）或 Kappa 统计量 -- **ICC 阈值**：ICC > 0.75 = 良好，0.60-0.75 = 中等，< 0.60 = 差 -- **盲法要求**：评分者必须不知道样本分组（治疗组 vs 对照组） -- **一致性人数**：至少 2-3 名经过培训的评分者 - -**关键问题**： -- 论文是否报告了评分者间一致性？ -- 评分者是否盲于实验分组？ -- 如果评分差异过大（如 >2 分），如何处理？（共识评分？平均？排除离群值？） - -### 3. ROI（Region of Interest）选择 - -- **评分位置**：是在修复区中心？还是包含修复区+周围正常软骨？ -- **切片位置**：是否使用最佳切片（无折叠、无撕裂）？ -- **染色一致性**：同批次染色各组切片条件是否相同？ - -### 4. 软骨特异性染色评估要点 - -| 染色类型 | 评估内容 | 关键指标 | -| ------------------ | ------------------------------- | --------------------------------- | -| **Safranin O / Fast Green** | 蛋白聚糖（PG）含量 | 染色强度（0-3 或 0-4 分制）；与正常软骨对比 | -| **H&E** | 细胞形态、结构完整性 | 细胞数、细胞簇、核形态 | -| **Masson Trichrome** | 胶原沉积与纤维化程度 | 蓝绿色胶原 vs 红色肌肉/细胞质 | -| **番红O（Safranin O）** | PG 分布 | 深层/浅层分布是否与正常软骨相似 | -| **IHC（COL2A1等）** | 目标蛋白表达定位与强度 | 阳性染色区域百分比 | - -### 5. 评分细节审查 - -- **分项分数**还是**总分**？分项分数是否都有报告？ -- **正常软骨对照**：是否同时评分了正常对照组（Sham）作为基准？ -- **时间点设计**：多个时间点（4w / 12w / 24w）的评分是否用同一标准？ - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -| ------------------ | ------------------------------------------ | ---------------------------------- | -| **评分者非盲** | 评分者知道分组信息，导致评分偏倚 | 方法中是否注明"blinded evaluation" | -| **评分者间一致性差**| ICC < 0.6，但作者仍报告组间差异显著 | 检查是否报告了 ICC 或 κ 值 | -| **ROI 选取不统一** | 不同样本选取的评分区域大小不一致 | 方法中是否明确了 ROI 标准 | -| **总分掩盖分项差异**| 两组总分相近但分项模式完全不同 | 检查分项分数而非只看总分 | -| **缺乏正常对照** | 没有正常软骨/假手术组作为基准 | 是否设置了 baseline 对照 | -| **组织伪影当作数据**| 切片折叠、气泡、染色假象被纳入评分 | 图像质量是否足以区分真实信号和伪影 | - ---- - -## 关键问题模板 - -1. "用的是哪个评分系统（O'Driscoll / ICRS II / Mankin）？该系统的分项评分是否都有报告，还是只报告了总分？" -2. "评分是否采用了盲法？评分者间一致性（ICC/κ）是多少？是否在报告前就确认了一致性？" -3. "ROI 的选择标准是什么？是固定位置还是基于修复区中心？不同样本的评分区域是否可比较？" -4. "是否存在评分过高/过低的离群样本？作者如何处理离群值？" - -## 一句话总结 - -> 组织学评分的核心是**"查评分系统、验盲法执行、核ROI一致、看完总分看分项"**——半定量评分的主观性决定了盲法和一致性报告是可信度的基础。 diff --git a/skills/literature-qa/chart-reading/网络图与通路图.md b/skills/literature-qa/chart-reading/网络图与通路图.md deleted file mode 100644 index 2fa62850..00000000 --- a/skills/literature-qa/chart-reading/网络图与通路图.md +++ /dev/null @@ -1,78 +0,0 @@ -# 网络图与通路图 - -## 适用场景 - -展示分子相互作用网络（PPI）、信号通路级联、基因调控关系、或共表达网络。常见于系统生物学、多组学整合分析、药物靶点预测。 - -## 读图维度 - -### 1. 网络拓扑结构 - -- **节点 (Node)**： - - 代表什么？（基因、蛋白、代谢物、疾病、药物） - - 节点大小/颜色编码什么？（表达量、重要性、分类、差异显著性） - - 关键节点（Hub）是否被标注？ -- **边 (Edge)**： - - 代表什么关系？（物理相互作用、调控、共表达、预测关联） - - 边的粗细/颜色编码什么？（置信度、强度、关系类型） - - 是否有方向性？（箭头表示调控方向） - -### 2. 网络布局算法 - -- **布局方式**： - - Force-directed（力导向）：相关节点聚集，无关节点分离 - - Circular（环形）：强调层次结构 - - Hierarchical（层次）：展示级联信号 -- **布局选择影响**： - - 不同算法可能强调不同的网络特征 - - 聚集的模块是否代表真实功能单元？ - -### 3. 模块与聚类 - -- **模块检测**： - - 是否有明显的社区/模块结构？ - - 模块用什么算法识别？（MCL、Louvain、WGCNA） - - 模块内节点是否有共享功能？ -- **通路注释**： - - 节点是否标注了通路归属？（如 KEGG、GO、Reactome） - - 同一通路的节点是否在网络中聚集？ - -### 4. 中心性指标 - -- **Hub 基因/蛋白**： - - Degree（连接数）最高的节点是谁？ - - Betweenness（中介中心性）高的节点 = 信息流通的关键瓶颈 - - Closeness（接近中心性）高的节点 = 网络核心 -- **Hub 的生物学意义**： - - 是否是已知的"明星分子"？ - - 是否是潜在的药物靶点？ - -### 5. 多组学整合层 - -- **多层网络**： - - 不同颜色/形状的节点是否代表不同组学层面？（基因、蛋白、代谢物） - - 层间连接（如基因→蛋白）是否可靠？ -- **整合可信度**： - - 跨层关联是否有实验验证？ - - 还是纯计算预测？ - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| ** hairball 效应** | 节点>100，边极度密集 | 无法解读，视觉噪声 | -| **预测边混入** | 未区分实验验证 vs 预测 | 假阳性关联污染网络 | -| **Hub 偏倚** | 只关注已知明星分子 | 忽视真正有功能的新节点 | -| **布局artifact** | 节点聚集仅因布局算法 | 被误读为生物学模块 | -| **过度解读方向性** | 蛋白相互作用标为"激活" | PPI 本身不携带方向信息 | - -## 关键问题模板 - -1. "网络图中包含 200+ 节点和 1000+ 边，是否进行了阈值过滤？未显示的边是否可能改变模块结构？" -2. "标注的 Hub 节点 Degree 最高，但 Betweenness 低，它是否是真正的功能核心还是仅仅连接了很多边缘节点？" -3. "网络中的边有 30% 来自 STRING 预测（非实验验证），如果仅保留实验验证的边，网络结构是否崩溃？" -4. "通路图显示 A 激活 B 激活 C，但这是基于文献挖掘的通路模板，本研究的数据是否支持这种级联关系？" - -## 一句话总结 - -> 网络图的核心是**"看节点、看模块、看Hub、防 hairball"**——网络的可读性和边的可信度比节点的数量更重要。 diff --git a/skills/literature-qa/chart-reading/蛋白质结构图.md b/skills/literature-qa/chart-reading/蛋白质结构图.md deleted file mode 100644 index ebe4f79a..00000000 --- a/skills/literature-qa/chart-reading/蛋白质结构图.md +++ /dev/null @@ -1,79 +0,0 @@ -# 蛋白质结构图 - -## 适用场景 - -展示蛋白质三维结构、结构域组织、配体结合位点、突变位置、或蛋白-蛋白/蛋白-配体相互作用界面。常见于结构生物学、药物设计、突变功能预测。 - -## 读图维度 - -### 1. 表示方式与分辨率 - -- **结构表示法**： - - **Cartoon/Ribbon**：展示二级结构（α-螺旋、β-折叠、无规卷曲） - - **Surface**：展示表面电势、疏水性、可及性 - - **Stick/Ball-and-stick**：展示小分子配体、关键残基侧链 - - **Sphere**：展示金属离子、水分子 -- **分辨率 (Resolution)**： - - X-ray: < 2.5Å 为高质量，2.5-3.5Å 中等，>3.5Å 较低 - - Cryo-EM: < 3.0Å 原子级，3-5Å 可看清二级结构 - - 低分辨率结构的侧链位置不可靠 - -### 2. 结构域与功能域 - -- **结构域划分**： - - 是否标注了已知的功能域？（如激酶域、DNA结合域） - - 不同颜色是否代表不同结构域？ -- **无序区域 (IDR)**： - - N端或C端是否有未解析区域（虚线表示）？ - - 无序区域是否有功能重要性？ - -### 3. 突变与修饰位点 - -- **突变标注**： - - 突变位点在结构上的位置？（表面 vs 核心） - - 是否参与相互作用界面？ - - 是保守位点还是可变位点？（看序列比对） -- **翻译后修饰**： - - 磷酸化、泛素化、乙酰化位点是否标注？ - - 修饰位点是否暴露在表面？ - -### 4. 相互作用界面 - -- **蛋白-蛋白相互作用 (PPI)**： - - 界面面积多大？（>1500Ų 为典型强相互作用） - - 界面是否涉及疏水核心 + 极性边缘？ - - 关键残基是否通过 Stick 显示？ -- **蛋白-配体相互作用**： - - 配体结合口袋位置？（正构 vs 别构） - - 氢键、疏水作用、π-π堆积是否标注？ - - 结合亲和力 (Kd/Ki) 是否与结构互补性一致？ - -### 5. 动态性与构象变化 - -- **静态结构局限**： - - 单张结构图是否能代表动态构象系综？ - - 是否有 NMR 系综或 MD 模拟补充？ -- **构象变化**： - - 是否叠加了多种构象（如开放/关闭状态）？ - - 变化发生在哪个结构域？ - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **低分辨率过度解读** | Resolution > 4Å | 侧链位置不准确，界面分析不可靠 | -| **同源建模未标注** | 未说明是实验结构还是预测 | 将预测结构等同于实验验证 | -| **静态冻结动态** | 仅展示单一构象 | 忽视蛋白质天然动态性 | -| **修饰位点埋藏** | 标注的修饰位点在结构核心 | 修饰酶无法接触，标注可能错误 | -| **人工构建域** | 截短体或融合蛋白结构 | 不代表全长蛋白的真实状态 | - -## 关键问题模板 - -1. "该蛋白质结构分辨率为 3.8Å，作者声称突变破坏了氢键网络，但在这个分辨率下侧链取向是否足够精确以支持这一结论？" -2. "结构图显示配体结合在口袋 A，但生物化学实验显示主要活性来自口袋 B，晶体结构是否捕获了非生理相关的结合模式？" -3. "图中标注的磷酸化位点 Ser123 在结构上位于β-折叠核心内部，激酶如何接触这个位点？是否只在未折叠状态下修饰？" -4. "这是同源建模结构（基于 30% 序列同源的模板），而非实验解析结构，模型质量评分（如 QMEAN、MolProbity）是否在可接受范围？" - -## 一句话总结 - -> 蛋白质结构图的核心是**"看分辨率、看界面、看动态、防过度解读"**——结构是功能的快照，不是功能的全部。 diff --git a/skills/literature-qa/chart-reading/降维图(PCA-tSNE-UMAP).md b/skills/literature-qa/chart-reading/降维图(PCA-tSNE-UMAP).md deleted file mode 100644 index 797bd8ea..00000000 --- a/skills/literature-qa/chart-reading/降维图(PCA-tSNE-UMAP).md +++ /dev/null @@ -1,65 +0,0 @@ -# 降维图（PCA / t-SNE / UMAP）阅读指南 - -## 适用场景 -- 高维数据的可视化（RNA-seq、质谱、影像特征等） -- 样本相似性/异质性的初步探索 -- 批次效应检测 - ---- - -## 核心阅读维度 - -### 1. 方差解释率（PCA特有） -- **PC1/PC2百分比**：前两维解释了多少总方差？ - - >50%：降维充分，可信 - - 20-50%：中等，可能丢失信息 - - <20%：降维不充分，两维不足以代表数据结构 -- **碎石图（Scree Plot）**：如果有，看拐点在哪里。拐点后的PC贡献小，可忽略 - -### 2. 分组分离度 -- **视觉分离**：不同颜色/形状的组是否明显分开？ -- **重叠程度**：重叠多 = 组间差异小，或降维丢失了区分信息 -- **边界清晰度**：边界是清晰还是模糊？（模糊 = 组间有过渡状态） - -### 3. 批次效应检测 -- **批次标记**：如果有按批次着色，看同一批次样本是否聚集？ - - 是 = 存在批次效应，需要校正（如ComBat） - - 否 = 批次效应小，可喜 -- **与生物学分组的关系**：批次聚集和生物学分组哪个更强？ - - 如果批次 > 生物学 = 严重问题，结论不可信 - -### 4. 离群样本 -- **远离主群的点**：是否总是同一个样本？ -- **原因**：技术错误（如测序深度低）还是真实生物学异常？ -- **处理**：作者是否排除了离群点？排除标准是什么？ - -### 5. 降维方法差异 - -| 方法 | 优点 | 缺点 | 适用场景 | -|------|------|------|---------| -| PCA | 线性、可解释、有载荷 | 只能捕捉线性关系 | 初步探索、批次检测 | -| t-SNE | 非线性、局部结构好 | 随机、全局结构差、perplexity敏感 | 亚型识别、局部聚类 | -| UMAP | 非线性、速度快、全局+局部 | 超参数敏感（n_neighbors, min_dist） | 大规模数据、单细胞 | - -**关键问题**：作者选择该方法的理由是否充分？（如：单细胞数据用UMAP合理，但普通RNA-seq用t-SNE可能过度） - ---- - -## 常见陷阱 - -| 陷阱 | 说明 | 如何识别 | -|------|------|---------| -| Perplexity选择不当 | t-SNE的perplexity影响聚类外观 | 方法部分是否说明perplexity值？通常5-50 | -| UMAP过聚合 | min_dist太小导致所有点粘在一起 | 看是否有独立的小簇，还是一个大 blob | -| 忽略方差解释率 | 只展示PC1/PC2但解释率<10% | 看caption或正文是否报告了% | -| 循环论证 | 用聚类结果定义组，然后在降维图上展示 | 问：颜色标注的组是如何定义的？是否独立于降维？ | - ---- - -## 批判性提问模板 - -1. "PC1/PC2 解释了多少方差？如果<30%，二维投影是否充分？" -2. "分组颜色是基于独立标签（如临床诊断）还是聚类结果？" -3. "是否存在批次效应？批次效应是否大于生物学效应？" -4. "离群样本是否被排除？排除标准是什么？" -5. "如果用PCA而不是t-SNE/UMAP，分离度是否仍然明显？"（检验非线性方法的必要性） diff --git a/skills/literature-qa/chart-reading/雷达图与漏斗图.md b/skills/literature-qa/chart-reading/雷达图与漏斗图.md deleted file mode 100644 index 3c5cf32a..00000000 --- a/skills/literature-qa/chart-reading/雷达图与漏斗图.md +++ /dev/null @@ -1,84 +0,0 @@ -# 雷达图与漏斗图 - -## 适用场景 - -**雷达图**：多维度比较（如不同样本的多指标特征画像），常见于多参数免疫分型、药物多靶点评估、患者多维特征比较。 - -**漏斗图**：Meta分析中检测发表偏倚，或展示临床流程中各阶段的病例流失（如从筛查到入组到完成随访）。 - -## 读图维度 - -### 雷达图 - -#### 1. 维度定义与尺度 - -- **每个轴代表什么指标？** - - 指标是否独立？（避免高度相关的指标重复） - - 指标是否经过标准化？（否则量纲大的指标会主导图形） - - 轴的方向：是否所有轴都是"越大越好"？（注意反向指标） - -#### 2. 多边形形态解读 - -- **整体面积**： - - 面积越大 = 综合得分越高（在标准化前提下） - - 但面积受维度数量影响（维度越多面积越容易大） -- **形状特征**： - - 某一轴特别突出 = 该维度是主要特征 - - 接近正多边形 = 各维度均衡 - - 明显不对称 = 存在优势/劣势维度 - -#### 3. 组间比较 - -- **多边形重叠**： - - 完全重叠 = 两组特征几乎相同 - - 某一方向明显分离 = 该维度是主要区分特征 -- **交叉情况**： - - A组在维度1上高于B组，但在维度2上低于B组 = 权衡关系 - -### 漏斗图 (Funnel Plot) - -#### 1. 坐标轴含义 - -- **X轴**：效应量（如 logHR、logOR、SMD） -- **Y轴**：精度指标（通常是标准误 SE 的倒数，即 1/SE） - - 上方 = 大样本、高精度研究 - - 下方 = 小样本、低精度研究 - -#### 2. 对称性判断 - -- **理想情况**： - - 呈倒漏斗形，大样本研究聚集在顶部中间 - - 小样本研究在底部对称分布 -- **发表偏倚信号**： - - 底部左侧或右侧缺失 = 小样本阴性结果未发表 - - 图形明显不对称 = 可能存在发表偏倚 - -#### 3. 统计检验 - -- **Egger 检验**： - - p < 0.05 提示存在发表偏倚 - - 但检验力低（研究数 < 10 时不稳定） -- **Trim and Fill**： - - 估算需要多少"缺失"研究才能使漏斗对称 - - 调整后的合并效应量是否仍显著？ - -## 常见陷阱 - -| 陷阱 | 识别方法 | 风险 | -|------|---------|------| -| **雷达图维度过多** | >8 个轴 | 可读性极差，图形混乱 | -| **雷达图未标准化** | 不同轴量纲差异巨大 | 大数值轴完全压制小数值轴 | -| **漏斗图研究过少** | n < 10 | 无法判断对称性，检验无功效 | -| **混淆两种漏斗图** | Meta漏斗 vs 临床流程漏斗 | 解读方式完全不同 | -| **假对称性** | 异质性高导致分散 | 异质性大时漏斗图本就不应集中 | - -## 关键问题模板 - -1. "雷达图有 12 个维度，是否有维度压缩或主成分分析支持这些维度的独立性？" -2. "漏斗图底部右侧明显缺失小样本阴性研究，如果进行 Trim and Fill 校正，合并效应量是否仍具有临床意义？" -3. "雷达图中样本 A 的面积大于样本 B，但 A 在关键疗效指标上反而低于 B，面积大小是否被非关键指标主导？" -4. "漏斗图显示不对称，但 I² = 85%（高异质性），这种分散是发表偏倚还是真实异质性导致？" - -## 一句话总结 - -> 雷达图看**"多维度均衡与失衡"**，漏斗图看**"小样本是否对称缺失"**——两者都警惕过度简化和维度误导。 diff --git a/skills/literature-qa/prompt_deep_subagent.md b/skills/literature-qa/prompt_deep_subagent.md deleted file mode 100644 index 5fbd6160..00000000 --- a/skills/literature-qa/prompt_deep_subagent.md +++ /dev/null @@ -1,298 +0,0 @@ -# Subagent Prompt for /pf-deep - -## 任务 - -对指定论文完成 journal-club 风格的精读（基于 Keshav 三阶段阅读法），写入正式文献笔记的 `## 🔍 精读` 区域。 - -## 输入变量（由主 agent 填入） - -- `{{ZOTERO_KEY}}` — Zotero key，如 `Y5KQ4JQ7` -- `{{VAULT}}` — Vault 根路径 -- `{{SCRIPT}}` — `//literature-qa/scripts/ld_deep.py` - -## 正确流程（必须按顺序执行） - -### 第一步：一键前置准备（运行 prepare 命令） - -**这是唯一需要运行的机械化命令**。它会自动完成：检查状态、生成 figure-map、扫描图表类型、插入骨架。 - -``` -python {{SCRIPT}} prepare {{ZOTERO_KEY}} --vault "{{VAULT}}" --format text -``` - -**期望输出示例**： -``` -[OK] Prepared Y5KQ4JQ7 - Formal note: Y5KQ4JQ7 - Title.md - Figures: 6 | Tables: 2 - Chart guides: 5 recommended -``` - -**如果输出以 `[ERROR]` 开头**： -- 立即停止，将错误信息报告给主 agent -- 常见错误：analyze != true（需用户在 Base 中勾选 analyze）、OCR 未完成、formal note 不存在 - -**如果输出 `[WARN] deep_reading_status already 'done'`**： -- 这说明该论文已经精读过。如果用户要求重新精读，通知主 agent 确认是否覆盖。 - -**prepare 成功后会自动生成以下文件**： -- `/PaperForge/ocr/{{ZOTERO_KEY}}/figure-map.json` — 图表清单 -- `/PaperForge/ocr/{{ZOTERO_KEY}}/chart-type-map.json` — 图表类型与推荐指南 - -**Agent 需要读取 chart-type-map.json**，为每张 figure 建立 `chart_types` 备忘列表。在 Pass 2 解析该 figure 时，根据其 chart_types 读取 `{{CHART_READING_DIR}}` 下对应的 chart-reading 指南，将关键审查问题整合进"图表质量审查"段落。 - -> **注意**：prepare 命令已自动在 formal note 中插入了 `## 🔍 精读` 骨架（包含所有 figure/table 的 callout 块）。Agent **不需要**再手动运行 ensure-scaffold。 - ---- - -### 第二步：Pass 1 概览（单独执行，完成后保存） - -**执行策略**：Pass 1 只填写 `### Pass 1: 概览` 区域，不要碰 Pass 2/3 的内容。 - -用 Read 工具加载 formal note（路径在 prepare 输出中），找到 `## 🔍 精读` 区域，定位到 `### Pass 1: 概览`，将占位符替换为实际内容。 - -快速扫描，建立全局认知，决定是否值得深入。 - -```markdown -### Pass 1: 概览 - -**一句话总览** -（论文类型 + 核心贡献一句话，如：这是一项关于XX的前瞻性队列研究，核心发现是YY。） - -**5 Cs 快速评估** -- **Category**（类型）：这是什么类型的论文？测量/分析/原型/方法学/综述/临床试验/队列研究/病例系列？ -- **Context**（上下文）：基于哪些理论或前人工作？与哪些论文直接相关？理论基础是什么？ -- **Correctness**（合理性初判）：作者的假设看起来合理吗？样本量/研究设计是否匹配研究问题？（第一遍只做直觉判断，后续 Pass 3 深入质疑） -- **Contributions**（贡献）：明确列出 1-3 个主要贡献。 -- **Clarity**（清晰度）：论文结构是否合理？写作是否清晰？图表是否自明？ - -**Figure 导读（快速扫图，仅列编号 + 一页纸猜测）** -- 关键主图：Figure 1（猜测：XXX）、Figure 2（猜测：YYY）... -- 证据转折点：预计 Figure X 是核心结论的支撑 -- 需要重点展开的 supplementary：（如有） -- 关键表格：Table X（猜测：ZZZ） -``` - -**完成后**：用 Edit 工具将填写好的 Pass 1 内容写回 formal note，保存。然后继续下一步。 - ---- - -### 第三步：Pass 2 精读还原（分块执行，每块完成后保存） - -**执行策略**：Pass 2 只填写 `### Pass 2: 精读还原` 区域。内容量大，必须分块执行，不要试图一次写完全部 figure。 - -#### 分块规则 - -将 figure/table 分为 2-3 个一组的块： -- 第 1 块：Figure 1-2（或 1-3） -- 第 2 块：Figure 3-4（或 4-5） -- 第 3 块：剩余 Figure + Table + 关键方法补课 + 主要发现与新意 - -每完成一个块，**立即用 Edit 保存**，然后 Read 确认已写入，再继续下一个块。 - -#### Figure-by-Figure 解析（逐块填写） - -每张主图按以下结构： - -```markdown -##### Figure N：{caption 一句话概括} -![[image_link]] - -**图像定位与核心问题** -- 页码： -- 这张图要回答什么： - -**方法与结果** -- 方法：（实验设计/数据来源/技术路线） -- 结果：（图中展示的核心数据点/趋势/对比） - -**图表类型识别与 chart-reading 引用（强制）** -1. **识别子图类型**：基于 caption 和图像内容，列出该 figure 包含的所有图表子类型（如：柱状图、折线图、免疫荧光图、热图等） -2. **读取 chart-reading 参考**：根据 prepare 生成的 chart-type-map.json 中该 figure 的 `recommended_guides`，读取对应指南文件 -3. **执行审查清单**：将 chart-reading 指南中的核心审查问题逐条应用到该 figure 上，至少回答以下问题： - - 如果是**柱状图/条形图**：Y轴是否从0开始？误差棒类型是SD/SEM/CI？是否进行了多重比较校正？ - - 如果是**折线图/时间序列**：曲线是否符合某种动力学模型？是否达到平台期？突释效应如何？ - - 如果是**免疫荧光图**：是否使用 sequential scanning？荧光强度定量方法是什么？背景阈值如何设定？ - - 如果是**热图/聚类图**：标准化方法是什么（Z-score/log/percentile）？聚类算法和距离度量是什么？是否存在循环论证？ - - 如果是**火山图**：显著性阈值是raw p-value还是FDR？效应量阈值是多少？上调/下调是否对称？ - - 如果是**MSEA/GSEA富集图**：q值阈值是多少（0.25还是0.05）？基因集大小是否合理？Leading Edge占比多少？ - - 如果是**弦图/网络图**：弦的粗细代表什么？是否过度解读了方向性？ - - 如果是**组织学图**：评分系统是什么？是否盲法评分？评分者间一致性（ICC/κ）是否报告？ - - 如果是**箱式图/小提琴图**：中位数、IQR、异常值如何？组间分布是否对称？ - - 如果是**散点图/气泡图**：相关性系数是多少？是否进行了回归分析？R²值如何？ - - 如果是**ROC曲线**：AUC是多少？置信区间是否报告？截断点如何选择？ - - 如果是**生存曲线**：中位生存期是多少？HR和置信区间是否报告？删失数据如何处理？ -4. **整合审查结果**：将上述审查的发现写入下方的 "**图表质量审查**" 段落。如果某条审查不适用，明确标注"N/A"；如果审查发现问题，用 `> [!warning]` 标出。 - -**图表质量审查** -- 轴标签是否完整？单位是否标注？ -- 是否有 error bars / 置信区间 / 统计显著性标记？ -- 根据 chart-reading 指南执行后的额外发现： -- 如果缺少这些，对结论可信度有什么影响？ - -**作者解释** -- 作者在文中对该图的描述： - -**我的理解** -- 自己的分析（区分"作者解释"和"我的理解"）： - -**在全文中的作用** -- 该图在整体故事线中的位置： - -**疑点 / 局限** -- （可酌情用 `> [!warning]` 突出） -``` - -#### Table-by-Table 解析 - -每张重要表格按以下结构： - -```markdown -##### Table N：{caption 一句话概括} -![[image_link]] - -- 这张表在回答什么问题： -- 关键字段 / 分组： -- 主要结果： -- 我的理解： -- 在全文中的作用： -- 疑点 / 局限： -``` - -#### 关键方法补课 -- 方法 1：（如有不熟悉的实验技术，简要补课） -- 方法 2： - -#### 主要发现与新意 -**主要发现** -- 发现 1：（证据来源：Figure X / Table Y） -- 发现 2： - -**完成后**：确保 Pass 2 所有内容都已保存到 formal note，然后继续下一步。 - ---- - -### 第四步：Pass 3 深度理解（基于已写内容，完成后保存） - -**执行策略**：Pass 3 填写 `### Pass 3: 深度理解` 区域。这是基于前两个 pass 已写入内容的总结与升华，可以引用 Pass 1/2 中的具体发现。 - -对医学文献，Pass 3 的核心不是"复现"，而是**批判性评估 + 临床/研究迁移**。可以 spawn 多个 subagent 并行分析不同维度，最后整合。 - -**并行分析维度（可选，根据论文类型选择 2-3 个）：** - -1. **临床证据质量评估**：循证医学视角（PICO、偏倚风险、证据等级） -2. **方法学审查**：统计方法、实验设计、样本量、对照设置 -3. **领域上下文对比**：与当前领域最新进展的关系，是否被后续研究验证或推翻 -4. **研究迁移思考**：如果把这个方法/结论用到我的课题上，需要什么条件？最大障碍是什么？ - -```markdown -### Pass 3: 深度理解 - -#### 假设挑战与隐藏缺陷 -- **隐含假设**：（作者未明说但依赖的假设，如"样本代表性""测量无偏""模型线性"） -- **如果放宽某个假设，结论还成立吗？** -- **缺少哪些关键引用？**（相关工作是否充分对比？） -- **实验/分析技术的潜在问题**：（样本量、对照组、盲法、统计方法选择、混杂因素控制） - -#### 哪些结论扎实，哪些仍存疑 -**较扎实** -- ... - -**仍存疑** -- ... - -#### Discussion 与 Conclusion 怎么读 -- 作者真正完成了什么： -- 哪些地方有拔高： -- 哪些地方是推测： - -#### 对我的启发 -- 研究设计上： -- figure 组织上： -- 方法组合上： -- **未来工作想法**： - -#### 遗留问题 -**遗留问题** -- ... -``` - -**关键提示**：Pass 3 的写作可以引用 Pass 1 的"5 Cs 评估"和 Pass 2 的"主要发现"作为基础，形成连贯的批判性分析。不要孤立地写 Pass 3。 - -**完成后**：用 Edit 保存，然后进入验证步骤。 - ---- - -### 第五步：Callout 使用规则 - -填写时选择性使用 callout 突出重要信息： -- **主要发现**的每条 → `> [!important]` -- **仍存疑**的每条 → `> [!warning]` -- **遗留问题** → `> [!question]` -- **证据边界说明** → `> [!warning]` -- **疑点/局限** → `> [!warning]` -- 常规结构节（研究问题、路线、方法、启发等）保持普通 Markdown 列表，不用 callout - -**Callout 间距强制要求**： -- 多个 `> [!important]`、`> [!warning]` 或 `> [!question]` 之间**必须有空行**，否则 Obsidian 会合并成一个 callout。 -- Figure/Table 的 `> [!note]-` callout 之间也**必须有空行**。 - -**错误示范（太乱 + 会合并）**： -``` -> [!important] 主要发现 -> 发现 1：... -> 发现 2：... -> [!warning] 仍存疑 -> - ... -``` - -**正确示范（简洁 + 有空行）**： -``` -**主要发现** -> [!important] 发现 1：（文字内容） - -> [!important] 发现 2：（文字内容） - -**仍存疑** -> [!warning] - 某结论缺乏独立验证 -``` - ---- - -### 第六步：验证完整性 - -完成后，运行 validate-note 检查结构完整性： - -``` -python {{SCRIPT}} validate-note "" --fulltext "" -``` - -其中 `` 和 `` 从 prepare 的输出中获取。 - -如果输出不是 `OK`，说明有缺失的 section headings 或 figure embeds，需要修复后再报完成。 - -### 错误处理 - -- 如果 `prepare` 返回 `[ERROR]`：立即停止，将完整错误信息报告给主 agent -- 如果 `validate-note` 失败：列出缺失项目并修复。**注意**：`validate-note` 会检查 callout 间距问题——如果多个 `> [!warning]` / `> [!question]` / `> [!note]-` 之间缺少空行，会报告 spacing 错误。修复方法：在相邻 callout 之间插入空行。 - -## 交付要求 - -1. prepare 成功后，报告：formal note 路径、figure 数量、table 数量 -2. 所有内容填写完毕后，运行 validate-note 并报告结果 -3. 如有任何异常，必须报告完整错误信息，不要静默跳过 -4. 只写回 formal note，不要写其他文件 - -## 参考：ld_deep.py 命令速查 - -```bash -# 一键前置准备（唯一需要 Agent 运行的机械化命令） -python {{SCRIPT}} prepare {{ZOTERO_KEY}} --vault "{{VAULT}}" --format text - -# 验证骨架完整性（完成所有 Pass 后运行） -python {{SCRIPT}} validate-note "" --fulltext "" - -# 列出待精读队列（信息用） -python {{SCRIPT}} queue --vault "{{VAULT}}" --format table -``` - diff --git a/skills/literature-qa/scripts/ld_deep.py b/skills/literature-qa/scripts/ld_deep.py deleted file mode 100644 index e7b25869..00000000 --- a/skills/literature-qa/scripts/ld_deep.py +++ /dev/null @@ -1,1420 +0,0 @@ -#!/usr/bin/env python3 -"""Helpers for /pf-deep literature sessions.""" - -from __future__ import annotations - -import argparse -import json -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Iterable - - -def _load_vault_config(vault: Path) -> dict: - """Load vault directory configuration — delegates to shared resolver. - - Preserves the public name for legacy callers. - """ - from paperforge.config import load_vault_config as _shared_load_vault_config - return _shared_load_vault_config(vault) - - -def _paperforge_paths(vault: Path) -> dict[str, Path]: - """Build PaperForge path inventory for /pf-deep — delegates to shared resolver. - - Returns ocr, records, literature keys matching shared resolver output. - """ - from paperforge.config import paperforge_paths as _shared_paperforge_paths - - shared = _shared_paperforge_paths(vault) - return { - "ocr": shared["ocr"], - "records": shared["library_records"], - "literature": shared["literature"], - } - - -def _get_ocr_root(vault: Path) -> Path: - """Get OCR root path dynamically from config.""" - return _paperforge_paths(vault)["ocr"] - - -STUDY_HEADER = "## 🔍 精读" -FIGURE_SECTION_HEADER = "#### Figure-by-Figure 解析" -TABLE_SECTION_HEADER = "#### Table-by-Table 解析" - - -@dataclass -class FigureEntry: - number: str - image_id: str - title: str - image_link: str - page: int | None - caption: str - is_supplementary: bool - additional_images: list[dict] = field(default_factory=list) - - -@dataclass -class TableEntry: - number: str - image_link: str - page: int | None - - -def validate_extraction_completeness( - figures: list[FigureEntry], - tables: list[TableEntry], - fulltext: str, -) -> list[str]: - """Check if figure/table extraction is complete and flag issues. - - Returns a list of warning messages for the agent. - """ - warnings: list[str] = [] - - # Check for unmapped figures (number="?") - unmapped_figures = [f for f in figures if f.number == "?"] - if unmapped_figures: - warnings.append( - f"[!WARNING] {len(unmapped_figures)} 个 figure 未从 figure-map 解析到编号，" - f"已标记为 '?'。Agent 需手动核对并补充真实编号（如 Fig 1, Fig 2...）。" - ) - - # Check for low figure count (Nature/Science papers typically have 4-8 figures) - main_figures = [f for f in figures if not f.is_supplementary] - if len(main_figures) < 2: - warnings.append( - f"[!WARNING] 仅提取到 {len(main_figures)} 个主图，数量异常偏低。" - f"请检查 OCR 质量或手动从 PDF 补充缺失的 figure。" - ) - - # Check for tables that might have been missed - # Look for table captions in fulltext that don't have corresponding images - table_caption_pattern = re.compile( - r"(?:Table|Extended Data Table|Supplementary Table)\s+(\d+)", - re.IGNORECASE, - ) - caption_numbers = set() - for match in table_caption_pattern.finditer(fulltext): - caption_numbers.add(match.group(1)) - - extracted_numbers = {t.number for t in tables} - missing_tables = caption_numbers - extracted_numbers - if missing_tables: - warnings.append( - f"[!WARNING] 检测到 {len(missing_tables)} 个表格引用未匹配到图像：" - f"Table {', '.join(sorted(missing_tables))}。" - f"Agent 需手动从 PDF 提取这些表格。" - ) - - return warnings - - -EVIDENCE_GUARDRAIL = ( - "**证据边界**：区分三层信息：`论文结果`、`作者解释`、`我的理解/推断`。" - "不要把样本内观察直接写成普遍规律，不要把相关性写成因果，不要把未进入最终模型的指标写成已被稳定验证的联合诊断结论。" -) - -REQUIRED_SECTIONS = [ - "**一句话总览**", - "**证据边界**", - "**Figure 导读**", - "**主要发现**", - "**较扎实**", - "**仍存疑**", - "**遗留问题**", -] - - -def extract_figures_from_fulltext(fulltext: str, figure_map: dict | None = None) -> list[FigureEntry]: - """Extract ALL image blocks from OCR fulltext markdown. - - Priority: use figure_map (from build_figure_map) if available for accurate - numbering and captions. Fallback to raw OCR image blocks with '?' numbering. - - Returns all images with their page number and image basename. - The agent reads fulltext to identify which are real figures and their numbers, - then passes image basenames to ensure-scaffold via --figures. - """ - figures: list[FigureEntry] = [] - pending_page: int | None = None - pending_image: str | None = None - - page_pattern = re.compile(r"") - image_pattern = re.compile(r"!\[\[(.+?)\]\]") - - # Build lookup from figure_map if available - map_lookup: dict[str, dict] = {} - if figure_map: - for entry in figure_map.get("figures", []): - map_lookup[entry.get("image_id", "")] = entry - for entry in figure_map.get("supplementary_figures", []): - map_lookup[entry.get("image_id", "")] = entry - - for raw_line in fulltext.splitlines(): - line = raw_line.strip() - - page_match = page_pattern.match(line) - if page_match: - pending_page = int(page_match.group(1)) - pending_image = None - continue - - image_match = image_pattern.match(line) - if image_match: - pending_image = image_match.group(1) - image_id = Path(pending_image).stem - - # Try figure_map first for accurate metadata - mapped = map_lookup.get(image_id) - if mapped: - figures.append( - FigureEntry( - number=mapped.get("number", "?"), - image_id=image_id, - title=mapped.get("caption", "")[:80] + "..." if len(mapped.get("caption", "")) > 80 else mapped.get("caption", "[Figure]"), - image_link=pending_image, - page=mapped.get("page", pending_page), - caption=mapped.get("caption", ""), - is_supplementary=mapped.get("type", "") == "supplementary_figure", - additional_images=mapped.get("additional_images", []), - ) - ) - else: - figures.append( - FigureEntry( - number="?", - image_id=image_id, - title="[OCR图像块]", - image_link=pending_image, - page=pending_page, - caption="", - is_supplementary=False, - ) - ) - continue - - return figures - - -def build_figure_plan( - figures: Iterable[FigureEntry], - important_supplementary: set[str] | None = None, -) -> list[FigureEntry]: - """Keep all main figures and only requested supplementary figures.""" - important_supplementary = important_supplementary or set() - planned: list[FigureEntry] = [] - for figure in figures: - if not figure.is_supplementary or figure.number in important_supplementary: - planned.append(figure) - return planned - - -def select_entries_by_numbers[T](entries: Iterable[T], numbers: set[str], attr: str = "number") -> list[T]: - """Select entries by their number field while preserving order.""" - if not numbers: - return [] - selected: list[T] = [] - for entry in entries: - if getattr(entry, attr) in numbers: - selected.append(entry) - return selected - - -def extract_tables_from_fulltext(fulltext: str, figure_map: dict | None = None) -> list[TableEntry]: - """Extract OCR table blocks from fulltext markdown. - - Supports multiple naming conventions: - - page_XXX_table_YYYY... (raw OCR) - - extended_data_table_N... (extended data) - - supplementary_table_N... (supplementary) - - Priority: use figure_map if available for accurate numbering. - """ - tables: list[TableEntry] = [] - pending_page: int | None = None - - page_pattern = re.compile(r"") - # Relaxed pattern: match any image path containing "table" in the filename - table_pattern = re.compile(r"!\[\[(.*table[^\]]*\.\w+)\]\]", re.IGNORECASE) - - # Build lookup from figure_map if available - map_lookup: dict[str, dict] = {} - if figure_map: - for entry in figure_map.get("tables", []): - map_lookup[entry.get("image_id", "")] = entry - for entry in figure_map.get("supplementary_tables", []): - map_lookup[entry.get("image_id", "")] = entry - - for raw_line in fulltext.splitlines(): - line = raw_line.strip() - if not line: - continue - - page_match = page_pattern.match(line) - if page_match: - pending_page = int(page_match.group(1)) - continue - - table_match = table_pattern.match(line) - if table_match: - image_link = table_match.group(1) - image_id = Path(image_link).stem - - # Try figure_map first - mapped = map_lookup.get(image_id) - if mapped: - tables.append(TableEntry( - number=mapped.get("number", str(len(tables) + 1)), - image_link=image_link, - page=mapped.get("page", pending_page), - )) - else: - # Fallback: cannot determine real table number from filename - # The number in filename (e.g. 75 in page_024_table_75...) is an OCR internal ID, NOT the table number - tables.append(TableEntry(number="?", image_link=image_link, page=pending_page)) - - return tables - - -# --------------------------------------------------------------------------- -# Chart Type Detection: per-figure subfigure type scanning -# --------------------------------------------------------------------------- - -CHART_TYPE_KEYWORDS: dict[str, list[str]] = { - "条形图与误差棒": [ - "bar", "histogram", "column", "quantification", "quantitative", - "relative expression", "mRNA expression", "protein level", - "BMD", "BV/TV", "Tb.N", "Tb.Sp", "intensity", "score", - "ratio", "percentage", "proportion", "fold change", - ], - "折线图与时间序列": [ - "curve", "time", "kinetics", "release", "cyclic", "stability", - "degradation", "profile", "over time", "day", "week", - "duration", "period", "cycle", "repeated", - ], - "热图与聚类图": [ - "heatmap", "heat map", "clustering", "cluster", "dendrogram", - "hierarchical", "correlation matrix", "expression matrix", - ], - "火山图与曼哈顿图": [ - "volcano", "volcano plot", "manhattan", "-log10", "log2fc", - "fold change", "differential", "DEG", "DEM", "upregulated", "downregulated", - ], - "免疫荧光定量图": [ - "immunofluorescence", "immunofluorescent", "confocal", "fluorescence", - "staining", "3D reconstruction", "overlay", "DAPI", " Alexa ", - "FITC", "TRITC", "Cy3", "Cy5", "SOX9", "COL2A1", "Piezo1", - "positive cells", "mean intensity", "fluorescent", - ], - "组织学半定量图": [ - "H&E", "hematoxylin", "eosin", "Safranin", "Fast Green", - "Masson", "trichrome", "histology", "histological", - "tissue section", "slide", "stain", "morphology", - "O'Driscoll", "ICRS", "Mankin", "OARSI", "Pineda", - ], - "桑基图与弦图": [ - "chord", "Sankey", "correlation network", "interaction network", - "signaling network", "proximity", "correlation of metabolic", - "Spearman", "Pearson correlation", - ], - "雷达图与漏斗图": [ - "radar", "funnel", "spider", "web chart", "polar", - ], - "GSEA富集图": [ - "GSEA", "MSEA", "enrichment", "enrichment plot", "enrichment score", - "NES", "leading edge", "pathway enrichment", "metabolic set", - ], - "箱式图与小提琴图": [ - "box", "boxplot", "box plot", "violin", "whisker", "quartile", - "median", "IQR", "outlier", - ], - "散点图与气泡图": [ - "scatter", "bubble", "dot plot", "correlation plot", "regression", - "linear fit", "R²", "correlation coefficient", - ], - "Western Blot条带图": [ - "Western", "blot", "WB", "gel electrophoresis", "SDS-PAGE", - "band", "molecular weight", "kDa", - ], - "显微照片与SEM图": [ - "SEM", "TEM", "microscopy", "micrograph", "morphology", - "surface", "topography", "nanostructure", "porous", - "FE-SEM", "scanning electron", "transmission electron", - ], - "降维图(PCA-tSNE-UMAP)": [ - "PCA", "t-SNE", "tSNE", "UMAP", "MDS", "dimensionality reduction", - "principal component", "clustering plot", "embedding", - ], - "网络图与通路图": [ - "pathway", "network", "signaling pathway", "KEGG", "Reactome", - "protein-protein interaction", "PPI", "regulatory network", - ], - "蛋白质结构图": [ - "protein structure", "crystallography", "NMR structure", "AlphaFold", - "3D structure", "homology modeling", "docking", - ], - "森林图与Meta分析": [ - "forest plot", "meta-analysis", "meta analysis", "pooled effect", - "heterogeneity", "I²", "Egger", "funnel plot", - ], - "ROC与PR曲线": [ - "ROC", "AUC", "receiver operating", "sensitivity", "specificity", - "PR curve", "precision-recall", "diagnostic accuracy", - ], - "生存曲线": [ - "survival", "Kaplan-Meier", "KM curve", "log-rank", "hazard ratio", - "HR", "OS", "PFS", "DFS", "mortality", - ], -} - - -def detect_chart_types(caption: str) -> list[str]: - """Detect which chart-reading guides are relevant for a figure caption. - - Returns a deduplicated list of chart type names (Chinese) that match - keywords found in the caption. Matching is case-insensitive. - """ - caption_lower = caption.lower() - matched: list[str] = [] - for chart_type, keywords in CHART_TYPE_KEYWORDS.items(): - for kw in keywords: - if kw.lower() in caption_lower: - matched.append(chart_type) - break # one match per chart type is enough - return matched - - -def build_chart_type_map(figure_map: dict) -> dict: - """Build a per-figure chart-type reference map from figure-map.json. - - Output format: - { - "zotero_key": "...", - "figures": [ - { - "number": "1", - "label": "Figure 1", - "detected_chart_types": ["雷达图与漏斗图"], - "recommended_guides": ["chart-reading/雷达图与漏斗图.md"] - }, - ... - ], - "tables": [...] - } - """ - result: dict = { - "zotero_key": figure_map.get("zotero_key", ""), - "figures": [], - "tables": [], - } - - for fig in figure_map.get("figures", []): - caption = fig.get("caption", "") - types = detect_chart_types(caption) - entry = { - "number": fig.get("number", "?"), - "label": fig.get("label", ""), - "caption_preview": caption[:200] + "..." if len(caption) > 200 else caption, - "detected_chart_types": types, - "recommended_guides": [f"chart-reading/{t}.md" for t in types], - } - result["figures"].append(entry) - - for tbl in figure_map.get("tables", []): - caption = tbl.get("caption", "") - types = detect_chart_types(caption) - entry = { - "number": tbl.get("number", "?"), - "label": tbl.get("label", ""), - "caption_preview": caption[:200] + "..." if len(caption) > 200 else caption, - "detected_chart_types": types, - "recommended_guides": [f"chart-reading/{t}.md" for t in types], - } - result["tables"].append(entry) - - return result - -CAPTION_PATTERNS = { - "main_figure": re.compile( - r"^(?:Figure|Fig\.?)\s*(\d+[a-zA-Z]?)(?:\s*[\.:|\-]?\s*)(.*?)$", - re.IGNORECASE, - ), - "supplementary_figure": re.compile( - r"^(?:Supplementary\s+(?:Figure|Fig\.?)\s*|Extended\s+Data\s+(?:Figure|Fig\.?)\s*|Suppl?\.?\s*(?:Figure|Fig\.?)\s*)(S?\d+[a-zA-Z]?)(?:\s*[\.:|\-]?\s*)(.*?)$", - re.IGNORECASE, - ), - "main_table": re.compile( - r"^(?:Table)\s*(\d+[a-zA-Z]?)(?:\s*[\.:|\-]?\s*)(.*?)$", - re.IGNORECASE, - ), - "supplementary_table": re.compile( - r"^(?:Supplementary\s+Table\s*|Suppl?\.?\s*Table\s*|Extended\s+Data\s+Table\s*)(S?\d+[a-zA-Z]?)(?:\s*[\.:|\-]?\s*)(.*?)$", - re.IGNORECASE, - ), -} - - -def build_figure_map(fulltext: str, zotero_key: str = "") -> dict: - """Build a caption-driven inventory of figures and tables. - - Scans fulltext.md line-by-line, detects formal captions, and pairs each - caption with the nearest image block within a window of adjacent pages. - """ - lines = fulltext.splitlines() - page_pattern = re.compile(r"") - image_pattern = re.compile(r"!\[\[(.+?)\]\]") - - # First pass: collect all images with their page and line index - all_images: list[dict] = [] - current_page: int | None = None - - for idx, raw_line in enumerate(lines): - line = raw_line.strip() - if not line: - continue - - page_match = page_pattern.match(line) - if page_match: - current_page = int(page_match.group(1)) - continue - - image_match = image_pattern.match(line) - if image_match: - all_images.append({ - "link": image_match.group(1), - "id": Path(image_match.group(1)).stem, - "line_idx": idx, - "page": current_page, - }) - - # Second pass: match captions to nearest images - entries: list[dict] = [] - current_page = None - - for idx, raw_line in enumerate(lines): - line = raw_line.strip() - if not line: - continue - - page_match = page_pattern.match(line) - if page_match: - current_page = int(page_match.group(1)) - continue - - # Try each caption pattern - for entry_type, pattern in CAPTION_PATTERNS.items(): - m = pattern.match(line) - if not m: - continue - number = m.group(1) - caption_text = m.group(2).strip() if len(m.groups()) > 1 else "" - - # Find nearest image within window of adjacent pages (current ± 2 pages) - best_image = None - min_distance = float('inf') - - for img in all_images: - # Check if image is within 2 pages of current page - if current_page is not None and img["page"] is not None: - page_diff = abs(img["page"] - current_page) - if page_diff <= 2: - distance = abs(img["line_idx"] - idx) - if distance < min_distance: - min_distance = distance - best_image = img - - # Find additional images on the same page near the best image - additional_images = [] - if best_image: - for img in all_images: - if img is best_image: - continue - if img["page"] == best_image["page"]: - # Check if this image is adjacent to best_image (within reasonable line distance) - line_diff = abs(img["line_idx"] - best_image["line_idx"]) - if line_diff <= 5: # Within 5 lines - additional_images.append({"id": img["id"], "link": img["link"]}) - - entries.append({ - "number": number, - "label": line.split(".")[0] if "." in line else line.split("|")[0].strip(), - "page": current_page, - "type": entry_type, - "caption": caption_text, - "image_link": best_image["link"] if best_image else None, - "image_id": best_image["id"] if best_image else None, - "additional_images": additional_images, - }) - break # one caption per line - - # Deduplicate by (type, number) keeping first - seen: set[tuple[str, str]] = set() - deduped: list[dict] = [] - for e in entries: - key = (e["type"], e["number"]) - if key not in seen: - seen.add(key) - deduped.append(e) - - return { - "zotero_key": zotero_key, - "generated_at": "", - "figures": [e for e in deduped if e["type"] == "main_figure"], - "tables": [e for e in deduped if e["type"] == "main_table"], - "supplementary_figures": [e for e in deduped if e["type"] == "supplementary_figure"], - "supplementary_tables": [e for e in deduped if e["type"] == "supplementary_table"], - } - - -def find_note_by_zotero_key(workspace_root: Path, zotero_key: str) -> Path | None: - """Resolve the formal literature note from the configured literature directory.""" - literature_root = _paperforge_paths(workspace_root)["literature"] - if not literature_root.exists(): - return None - - frontmatter_pattern = re.compile(rf'^\s*zotero_key:\s*"?{re.escape(zotero_key)}"?\s*$', re.MULTILINE) - for note_path in literature_root.rglob("*.md"): - try: - text = note_path.read_text(encoding="utf-8") - except UnicodeDecodeError: - text = note_path.read_text(encoding="utf-8", errors="ignore") - if frontmatter_pattern.search(text): - return note_path - return None - - -def render_study_scaffold(figures: Iterable[FigureEntry], tables: Iterable[TableEntry] | None = None) -> str: - """Render the Keshav three-pass deep-reading scaffold.""" - figure_list = list(figures) - table_list = list(tables or []) - - # Build figure blocks using the standard renderer for consistency - figure_blocks = [render_figure_block(fig) for fig in figure_list] - figure_section = "\n\n".join(figure_blocks) if figure_blocks else "##### Figure 待补充\n\n- 暂未从 OCR 中解析到可用主图。" - - # Build table blocks using the standard renderer for consistency - table_blocks = [render_table_block(table) for table in table_list] - table_section = "\n\n".join(table_blocks) if table_blocks else "- 暂未从 OCR 中解析到需单独展开的表格。" - - # Build completeness check prompt - figure_count = len(figure_list) - table_count = len(table_list) - completeness_prompt = ( - f"> [!warning] 图表完整性自查\n" - f"> 以下内容由脚本自动提取，请人工核对是否完整：\n" - f"> - **Figures**：提取到 {figure_count} 个图像块（含主图及补充材料图）\n" - f"> - **Tables**：提取到 {table_count} 个表格块\n" - f"> - **核对方法**：对照论文原文，检查是否所有 Figure / Table / Extended Data Figure / Supplementary Table 均已包含\n" - f"> - **如有遗漏**：请在下方直接补充对应的 figure/table 解析块，格式与现有块保持一致" - ) - - return ( - f"{STUDY_HEADER}\n\n" - f"{completeness_prompt}\n\n" - f"{EVIDENCE_GUARDRAIL}\n\n" - "### Pass 1: 概览\n\n" - "**一句话总览**\n" - "（待补充）\n\n" - "**5 Cs 快速评估**\n" - "- **Category**（类型）：\n" - "- **Context**（上下文）：\n" - "- **Correctness**（合理性初判）：\n" - "- **Contributions**（贡献）：\n" - "- **Clarity**（清晰度）：\n\n" - "**Figure 导读**\n" - "- 关键主图：\n" - "- 证据转折点：\n" - "- 需要重点展开的 supplementary：\n" - "- 关键表格：\n\n" - "### Pass 2: 精读还原\n\n" - f"{FIGURE_SECTION_HEADER}\n\n" - f"{figure_section}\n\n" - f"{TABLE_SECTION_HEADER}\n\n" - f"{table_section}\n\n" - "#### 关键方法补课\n" - "- 方法 1：\n" - "- 方法 2：\n\n" - "#### 主要发现与新意\n" - "**主要发现**\n" - "- 发现 1：\n" - "- 发现 2：\n\n" - "### Pass 3: 深度理解\n\n" - "#### 假设挑战与隐藏缺陷\n" - "- 隐含假设：\n" - "- 如果放宽某个假设，结论还成立吗？\n" - "- 缺少哪些关键引用？\n" - "- 实验/分析技术的潜在问题：\n\n" - "#### 哪些结论扎实，哪些仍存疑\n" - "**较扎实**\n" - "- \n\n" - "**仍存疑**\n" - "- \n\n" - "#### Discussion 与 Conclusion 怎么读\n" - "- 作者真正完成了什么：\n" - "- 哪些地方有拔高：\n" - "- 哪些地方是推测：\n\n" - "#### 对我的启发\n" - "- 研究设计上：\n" - "- figure 组织上：\n" - "- 方法组合上：\n" - "- 未来工作想法：\n\n" - "#### 遗留问题\n" - "**遗留问题**\n" - "- \n" - ) - - -def render_figure_block(figure: FigureEntry) -> str: - page_suffix = f"（第 {figure.page} 页）" if figure.page else "" - lines = [ - f"> [!note]- Figure {figure.number}：{figure.title}", - f"> ![[{figure.image_link}]]", - ] - # Add additional images for multi-image figures - for add_img in figure.additional_images: - lines.append(f"> ![[{add_img['link']}]]") - lines.extend([ - ">", - "> **图像定位与核心问题**", - f"> - 页码：{page_suffix or '待补充'}", - "> - 这张图要回答什么：", - "> - （待补充）", - ">", - "> **方法与结果**", - "> - 方法：", - "> - 结果：", - ">", - "> **作者解释**", - "> - （待补充）", - ">", - "> **我的理解**", - "> - （待补充）", - ">", - "> **在全文中的作用**", - "> - （待补充）", - ">", - "> **疑点 / 局限**", - "> - （待补充）", - ]) - return "\n".join(lines) + "\n\n" - - -def render_table_block(table: TableEntry) -> str: - page_suffix = f"第 {table.page} 页" if table.page else "待补充" - lines = [ - f"> [!note]- Table {table.number}", - f"> ![[{table.image_link}]]", - ">", - f"> - 图像定位：{page_suffix}", - "> - 这张表在回答什么问题：", - "> - 关键字段 / 分组：", - "> - 主要结果：", - "> - 我的理解：", - "> - 在全文中的作用：", - "> - 疑点 / 局限：", - ] - return "\n".join(lines) + "\n\n" - - -def validate_selected_blocks(note_text: str, figures: Iterable[FigureEntry], tables: Iterable[TableEntry] | None = None) -> list[str]: - """Return missing selected figure/table embeds that must be repaired before generation.""" - missing: list[str] = [] - for figure in figures: - # Use prefix matching for heading to accommodate custom titles - heading_prefix = f"> [!note]- Figure {figure.number}：" - embed = f"![[{figure.image_link}]]" - # Check if any line starts with the heading prefix and embed exists - has_heading = any(line.strip().startswith(heading_prefix) for line in note_text.splitlines()) - if not has_heading or embed not in note_text: - missing.append(f"Figure {figure.number}") - - for table in tables or []: - heading_prefix = f"> [!note]- Table {table.number}" - embed = f"![[{table.image_link}]]" - has_heading = any(line.strip().startswith(heading_prefix) for line in note_text.splitlines()) - if not has_heading or embed not in note_text: - missing.append(f"Table {table.number}") - - return missing - - -def validate_callout_structure(note_text: str, figures: Iterable[FigureEntry]) -> list[str]: - """Check that the note keeps the required small set of section markers.""" - missing: list[str] = [] - for marker in REQUIRED_SECTIONS: - if marker not in note_text: - missing.append(marker) - - for figure in figures: - figure_heading_prefix = f"> [!note]- Figure {figure.number}：" - # Find the line that starts with this prefix - figure_idx = -1 - for i, line in enumerate(note_text.splitlines()): - if line.strip().startswith(figure_heading_prefix): - figure_idx = note_text.find(line) - break - if figure_idx != -1: - # Find the next callout or heading after this figure block - figure_block = note_text[figure_idx:] - next_callout = figure_block.find("\n> [!note]-", 1) - next_heading = figure_block.find("\n##### ", 1) - next_section = figure_block.find("\n#### ", 1) - end_positions = [p for p in [next_callout, next_heading, next_section] if p != -1] - if end_positions: - figure_block = figure_block[:min(end_positions)] - required_local = [ - "**作者解释**", - "**我的理解**", - "**疑点 / 局限**", - ] - for marker in required_local: - if marker not in figure_block: - missing.append(f"{figure_heading_prefix}::{marker}") - - return missing - - -def validate_callout_spacing(note_text: str) -> list[str]: - """Check that consecutive callout blocks are separated by blank lines. - - In Obsidian, a callout block starts with `> [!type]` and continues as long - as subsequent lines start with `>`. An empty line or a non-`>` line ends - the block. If another `> [!type]` appears before the block is properly - ended, Obsidian merges them into a single callout instead of rendering - them as separate blocks. - """ - issues: list[str] = [] - lines = note_text.splitlines() - in_callout_block = False - - for idx, line in enumerate(lines): - stripped = line.strip() - is_callout_start = stripped.startswith("> [!") - is_callout_continuation = stripped.startswith("> ") or stripped == ">" - - if is_callout_start: - if in_callout_block: - issues.append( - f"Line {idx + 1}: Callout '{stripped[:60]}...' appears inside " - f"an ongoing callout block without a blank line separator." - ) - in_callout_block = True - elif not is_callout_continuation and stripped: - # Non-blank, non-callout line ends any callout block - in_callout_block = False - elif not stripped: - # Blank line ends the callout block - in_callout_block = False - - return issues - - -def validate_scaffold_residue(note_text: str) -> list[str]: - """Check for leftover scaffold template text that should have been removed. - - After the subagent fills the note, certain auto-generated instructional - callouts (e.g. "图表完整性自查") are no longer needed and should be - deleted. Their presence indicates incomplete cleanup. - """ - issues: list[str] = [] - residue_markers = [ - "图表完整性自查", - "以下内容由脚本自动提取", - "如有遗漏：请在下方直接补充", - ] - lines = note_text.splitlines() - for idx, line in enumerate(lines): - for marker in residue_markers: - if marker in line: - issues.append( - f"Line {idx + 1}: Scaffold residue detected: '{marker}'. " - "Remove this auto-generated instruction after verification." - ) - break # One issue per line is enough - return issues - - -def validate_redundant_headings(note_text: str) -> list[str]: - """Check for redundant heading + bold text pairs like: - - #### 遗留问题 - **遗留问题** - - The bold line duplicates the heading and should be removed. - """ - issues: list[str] = [] - lines = note_text.splitlines() - for idx in range(len(lines) - 1): - current = lines[idx].strip() - next_line = lines[idx + 1].strip() - # Match #### Heading followed by **Heading** - if current.startswith("#### ") and next_line.startswith("**") and next_line.endswith("**"): - heading_text = current[5:].strip() - bold_text = next_line[2:-2].strip() - if heading_text == bold_text: - issues.append( - f"Line {idx + 2}: Redundant bold text '{next_line}' " - f"duplicates heading '{current}'. Remove the bold line." - ) - return issues - - -def validate_deep_note( - note_text: str, - figures: Iterable[FigureEntry], - tables: Iterable[TableEntry] | None = None, -) -> list[str]: - """Run the full structural validation for a generated deep-reading note.""" - issues: list[str] = [] - - if STUDY_HEADER not in note_text: - issues.append(STUDY_HEADER) - if FIGURE_SECTION_HEADER not in note_text: - issues.append(FIGURE_SECTION_HEADER) - if TABLE_SECTION_HEADER not in note_text: - issues.append(TABLE_SECTION_HEADER) - - issues.extend(validate_selected_blocks(note_text, figures, tables)) - issues.extend(validate_callout_structure(note_text, figures)) - issues.extend(validate_callout_spacing(note_text)) - - issues.extend(validate_scaffold_residue(note_text)) - issues.extend(validate_redundant_headings(note_text)) - - required_headings = [ - "### Pass 1: 概览", - "### Pass 2: 精读还原", - "### Pass 3: 深度理解", - "#### Figure-by-Figure 解析", - "#### Table-by-Table 解析", - "#### 关键方法补课", - "#### 主要发现与新意", - "#### 假设挑战与隐藏缺陷", - "#### 哪些结论扎实，哪些仍存疑", - "#### Discussion 与 Conclusion 怎么读", - "#### 对我的启发", - "#### 遗留问题", - ] - for heading in required_headings: - if heading not in note_text: - issues.append(heading) - - return issues - - -def _ensure_section_with_blocks(note_text: str, header: str, blocks: list[str]) -> str: - if not blocks: - return note_text - - updated = note_text - if header not in updated: - updated = updated.rstrip() + f"\n\n{header}\n\n" - - for block in blocks: - heading = block.splitlines()[0] - if heading not in updated: - insertion_point = updated.find(header) - if insertion_point == -1: - updated = updated.rstrip() + f"\n\n{header}\n\n{block}\n" - else: - header_end = updated.find("\n", insertion_point) - if header_end == -1: - header_end = len(updated) - insert_at = header_end + 1 - updated = updated[:insert_at] + "\n" + block + "\n\n" + updated[insert_at:] - - return updated - - -def ensure_study_section(note_text: str, figures: Iterable[FigureEntry], tables: Iterable[TableEntry] | None = None) -> str: - """Append the study scaffold if it does not yet exist.""" - figure_list = list(figures) - table_list = list(tables or []) - if STUDY_HEADER in note_text: - updated = _ensure_section_with_blocks(note_text, FIGURE_SECTION_HEADER, [render_figure_block(fig) for fig in figure_list]) - updated = _ensure_section_with_blocks(updated, TABLE_SECTION_HEADER, [render_table_block(table) for table in table_list]) - return updated - - stripped = note_text.rstrip() - scaffold = render_study_scaffold(figure_list, table_list) - if stripped: - return f"{stripped}\n\n{scaffold}\n" - return f"{scaffold}\n" - - -def _read_json(path: Path) -> dict: - if not path.exists(): - return {} - try: - return json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError): - return {} - - -def prepare_deep_reading(vault: Path, zotero_key: str, force: bool = False) -> dict: - """Automate all mechanical pre-reading steps for /pf-deep. - - Returns a dict with: - - status: "ok" | "error" - - message: human-readable summary - - formal_note: path to the formal literature note - - fulltext_md: path to OCR fulltext - - figure_map: path to figure-map.json - - chart_type_map: path to chart-type-map.json - - figures: list of extracted figures - - tables: list of extracted tables - - chart_recommendations: list of recommended chart-reading guides - """ - result: dict = { - "status": "error", - "message": "", - "zotero_key": zotero_key, - "formal_note": None, - "fulltext_md": None, - "figure_map": None, - "chart_type_map": None, - "figures": [], - "tables": [], - "chart_recommendations": [], - } - - paths = _paperforge_paths(vault) - records_root = paths["records"] - literature_root = paths["literature"] - ocr_root = paths["ocr"] - - # 1. Find library-record - record_path: Path | None = None - domain: str | None = None - if records_root.exists(): - for domain_dir in records_root.iterdir(): - if not domain_dir.is_dir(): - continue - candidate = domain_dir / f"{zotero_key}.md" - if candidate.exists(): - record_path = candidate - domain = domain_dir.name - break - - if record_path is None: - # Fallback: search by zotero_key in frontmatter - for domain_dir in records_root.iterdir(): - if not domain_dir.is_dir(): - continue - for candidate in domain_dir.glob("*.md"): - text = candidate.read_text(encoding="utf-8") - if re.search(rf'^zotero_key:\s*"?{re.escape(zotero_key)}"?', text, re.MULTILINE): - record_path = candidate - domain = domain_dir.name - break - if record_path: - break - - if record_path is None: - result["message"] = f"[ERROR] Library record not found for zotero_key={zotero_key}" - return result - - record_text = record_path.read_text(encoding="utf-8") - - # 2. Check analyze flag - analyze_match = re.search(r'^analyze:\s*(true|false)$', record_text, re.MULTILINE) - if not analyze_match or analyze_match.group(1) != "true": - result["message"] = f"[ERROR] analyze != true in {record_path}. Set analyze: true first." - return result - - # 3. Check deep_reading_status - status_match = re.search(r'^deep_reading_status:\s*"?(.*?)"?$', record_text, re.MULTILINE) - dr_status = status_match.group(1).strip() if status_match else "pending" - if dr_status == "done" and not force: - result["message"] = f"[WARN] deep_reading_status already 'done'. Use --force to re-run." - return result - - # 4. Check OCR / fulltext availability - ocr_dir = ocr_root / zotero_key - fulltext_md = ocr_dir / "fulltext.md" - meta_path = ocr_dir / "meta.json" - - if not fulltext_md.exists(): - result["message"] = f"[ERROR] OCR fulltext not found: {fulltext_md}. Run OCR first." - return result - - ocr_status = "pending" - if meta_path.exists(): - meta = _read_json(meta_path) - ocr_status = str(meta.get("ocr_status", "pending")).strip().lower() - - if ocr_status != "done": - result["message"] = f"[ERROR] OCR status='{ocr_status}', not 'done'. Wait for OCR or check meta.json." - return result - - result["fulltext_md"] = str(fulltext_md) - - # 5. Find formal note - formal_note: Path | None = None - if literature_root.exists() and domain: - domain_dir = literature_root / domain - if domain_dir.exists(): - # Try exact match first - for candidate in domain_dir.glob("*.md"): - if candidate.name.startswith(f"{zotero_key} ") or candidate.name.startswith(f"{zotero_key} -"): - formal_note = candidate - break - # Fallback: search by frontmatter zotero_key - if formal_note is None: - for candidate in domain_dir.glob("*.md"): - text = candidate.read_text(encoding="utf-8") - if re.search(rf'^zotero_key:\s*"?{re.escape(zotero_key)}"?', text, re.MULTILINE): - formal_note = candidate - break - - if formal_note is None: - # Try global search - for candidate in literature_root.rglob("*.md"): - text = candidate.read_text(encoding="utf-8") - if re.search(rf'^zotero_key:\s*"?{re.escape(zotero_key)}"?', text, re.MULTILINE): - formal_note = candidate - break - - if formal_note is None: - result["message"] = f"[ERROR] Formal note not found in {literature_root}. Run sync --index first." - return result - - result["formal_note"] = str(formal_note) - - # Save original note content for rollback - original_note_text = formal_note.read_text(encoding="utf-8") - - # Track files for rollback - written_paths: list[Path] = [] - - try: - # 6. Run figure-map - figure_map_path = ocr_dir / "figure-map.json" - fulltext_text = fulltext_md.read_text(encoding="utf-8") - figure_map = build_figure_map(fulltext_text, zotero_key=zotero_key) - figure_map["generated_at"] = __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() - figure_map_path.parent.mkdir(parents=True, exist_ok=True) - figure_map_path.write_text(json.dumps(figure_map, ensure_ascii=False, indent=2), encoding="utf-8") - written_paths.append(figure_map_path) - result["figure_map"] = str(figure_map_path) - - # 7. Run chart-type-scan - chart_type_map_path = ocr_dir / "chart-type-map.json" - chart_type_result = build_chart_type_map(figure_map) - chart_type_map_path.write_text(json.dumps(chart_type_result, ensure_ascii=False, indent=2), encoding="utf-8") - written_paths.append(chart_type_map_path) - result["chart_type_map"] = str(chart_type_map_path) - - # Collect recommendations - all_guides: set[str] = set() - for fig in chart_type_result.get("figures", []): - for guide in fig.get("recommended_guides", []): - all_guides.add(guide) - for tbl in chart_type_result.get("tables", []): - for guide in tbl.get("recommended_guides", []): - all_guides.add(guide) - result["chart_recommendations"] = sorted(all_guides) - - # 8. Extract figures/tables and run ensure-scaffold - figure_candidates = extract_figures_from_fulltext(fulltext_text, figure_map) - table_candidates = extract_tables_from_fulltext(fulltext_text, figure_map) - planned_figures = build_figure_plan(figure_candidates) - planned_tables = table_candidates # Include all tables by default - - note_text = formal_note.read_text(encoding="utf-8") - updated = ensure_study_section(note_text, planned_figures, planned_tables) - formal_note.write_text(updated, encoding="utf-8") - - result["figures"] = [ - {"number": f.number, "image_id": f.image_id, "page": f.page, "title": f.title} - for f in planned_figures - ] - result["tables"] = [ - {"number": t.number, "page": t.page, "image_link": t.image_link} - for t in planned_tables - ] - - result["status"] = "ok" - result["message"] = ( - f"[OK] Prepared {zotero_key}\n" - f" Formal note: {formal_note}\n" - f" Fulltext: {fulltext_md}\n" - f" Figures: {len(planned_figures)} | Tables: {len(planned_tables)}\n" - f" Chart guides: {len(all_guides)} recommended" - ) - return result - - except Exception as exc: - # Rollback: delete partial files and restore note - for path in written_paths: - if path.exists(): - path.unlink() - formal_note.write_text(original_note_text, encoding="utf-8") - result["message"] = f"[ERROR] Prepare failed for {zotero_key}: {exc}. Rolled back changes." - return result - - -def scan_deep_reading_queue(vault: Path) -> list[dict]: - """Scan library-records for analyze=true + deep_reading_status!=done entries. - - Returns a list of dicts with keys: - - zotero_key, title, domain, analyze, deep_reading_status, ocr_status - """ - paths = _paperforge_paths(vault) - records_root = paths["records"] - ocr_root = paths["ocr"] - queue: list[dict] = [] - if not records_root.exists(): - return queue - - for domain_dir in records_root.iterdir(): - if not domain_dir.is_dir(): - continue - domain = domain_dir.name - for record_path in domain_dir.glob("*.md"): - text = record_path.read_text(encoding="utf-8") - - # Extract frontmatter fields - zotero_key_match = re.search(r'^zotero_key:\s*(.+)$', text, re.MULTILINE) - analyze_match = re.search(r'^analyze:\s*(true|false)$', text, re.MULTILINE) - status_match = re.search(r'^deep_reading_status:\s*"?(.*?)"?$', text, re.MULTILINE) - title_match = re.search(r'^title:\s*"?(.+?)"?$', text, re.MULTILINE) - - zotero_key = zotero_key_match.group(1).strip().strip('"').strip("'") if zotero_key_match else record_path.stem - is_analyze = analyze_match is not None and analyze_match.group(1) == "true" - dr_status = status_match.group(1).strip() if status_match else "pending" - title = title_match.group(1).strip().strip('"') if title_match else "" - - if not is_analyze or dr_status == "done": - continue - - # Check OCR status - meta_path = ocr_root / zotero_key / "meta.json" - ocr_status = "pending" - if meta_path.exists(): - meta = _read_json(meta_path) - ocr_status = str(meta.get("ocr_status", "pending")).strip().lower() - - queue.append({ - "zotero_key": zotero_key, - "domain": domain, - "title": title, - "deep_reading_status": dr_status, - "ocr_status": ocr_status, - }) - - # Sort: OCR done first, then by domain, then by key - queue.sort(key=lambda row: ( - 0 if row["ocr_status"] == "done" else 1, - row["domain"], - row["zotero_key"], - )) - return queue - - -import sys - -# Fix Windows console encoding for Unicode output -if sys.platform == "win32": - try: - sys.stdout.reconfigure(encoding="utf-8") - except AttributeError: - # Python < 3.7 fallback - import codecs - sys.stdout = codecs.getwriter("utf-8")(sys.stdout.buffer) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Helpers for /pf-deep note scaffolding") - subparsers = parser.add_subparsers(dest="command", required=True) - - figure_parser = subparsers.add_parser("figure-index", help="Extract figures from fulltext markdown") - figure_parser.add_argument("fulltext", type=Path) - - scaffold_parser = subparsers.add_parser("ensure-scaffold", help="Append a deep-reading scaffold to a note") - scaffold_parser.add_argument("note", type=Path) - scaffold_parser.add_argument("--fulltext", type=Path, help="Optional fulltext markdown used to build figure headings") - scaffold_parser.add_argument("--figures", help="Comma-separated selected figure numbers to insert") - scaffold_parser.add_argument("--tables", help="Comma-separated selected table numbers to insert") - - validate_parser = subparsers.add_parser("validate-selected", help="Validate selected figure/table embeds are present in the note") - validate_parser.add_argument("note", type=Path) - validate_parser.add_argument("--fulltext", type=Path, required=True, help="OCR fulltext markdown used to resolve figure/table embeds") - validate_parser.add_argument("--figures", help="Comma-separated selected figure numbers to validate") - validate_parser.add_argument("--tables", help="Comma-separated selected table numbers to validate") - - full_validate_parser = subparsers.add_parser("validate-note", help="Run the full structural validation for a deep-reading note") - full_validate_parser.add_argument("note", type=Path) - full_validate_parser.add_argument("--fulltext", type=Path, required=True, help="OCR fulltext markdown used to resolve figure/table embeds") - full_validate_parser.add_argument("--figures", help="Comma-separated selected figure numbers to validate") - full_validate_parser.add_argument("--tables", help="Comma-separated selected table numbers to validate") - - queue_parser = subparsers.add_parser("queue", help="List papers awaiting deep reading from library records") - queue_parser.add_argument("--vault", type=Path, required=True, help="Path to the vault root") - queue_parser.add_argument("--format", choices=["json", "table"], default="json", help="Output format") - - map_parser = subparsers.add_parser("figure-map", help="Build caption-driven figure/table map from OCR fulltext") - map_parser.add_argument("fulltext", type=Path, help="OCR fulltext markdown path") - map_parser.add_argument("--key", default="", help="Zotero key for output metadata") - map_parser.add_argument("--out", type=Path, help="Optional output JSON path (default: stdout)") - - chart_type_parser = subparsers.add_parser("chart-type-scan", help="Scan figure captions for chart types and recommend chart-reading guides") - chart_type_parser.add_argument("figure_map", type=Path, help="Path to figure-map.json generated by figure-map command") - chart_type_parser.add_argument("--out", type=Path, help="Optional output JSON path (default: stdout)") - - prepare_parser = subparsers.add_parser("prepare", help="One-click prepare all mechanical steps for deep reading") - prepare_parser.add_argument("zotero_key", help="Zotero citation key") - prepare_parser.add_argument("--vault", type=Path, required=True, help="Path to vault root") - prepare_parser.add_argument("--format", choices=["json", "text"], default="text", help="Output format") - prepare_parser.add_argument("--force", action="store_true", help="Force re-run even if deep_reading_status is done") - - args = parser.parse_args() - - if args.command == "prepare": - result = prepare_deep_reading(args.vault, args.zotero_key, force=args.force) - if args.format == "json": - print(json.dumps(result, ensure_ascii=False, indent=2)) - else: - print(result.get("message", "Unknown result")) - return 0 if result["status"] == "ok" else 1 - - if args.command == "figure-index": - fulltext = args.fulltext.read_text(encoding="utf-8") - for figure in extract_figures_from_fulltext(fulltext): - print(f"Figure {figure.number}\tid={figure.image_id}\tpage={figure.page}\timage={figure.image_link}\ttitle={figure.title}") - return 0 - - if args.command == "ensure-scaffold": - note_text = args.note.read_text(encoding="utf-8") - figures: list[FigureEntry] = [] - tables: list[TableEntry] = [] - if args.fulltext: - fulltext = args.fulltext.read_text(encoding="utf-8") - # Auto-load figure-map.json from same directory as fulltext if available - figure_map = None - figure_map_path = args.fulltext.parent / "figure-map.json" - if figure_map_path.exists(): - try: - figure_map = json.loads(figure_map_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError): - pass - figure_candidates = extract_figures_from_fulltext(fulltext, figure_map) - table_candidates = extract_tables_from_fulltext(fulltext, figure_map) - selected_figures = {part.strip() for part in (args.figures or "").split(",") if part.strip()} - selected_tables = {part.strip() for part in (args.tables or "").split(",") if part.strip()} - figures = ( - select_entries_by_numbers(figure_candidates, selected_figures, attr="image_id") - if selected_figures - else build_figure_plan(figure_candidates) - ) - tables = select_entries_by_numbers(table_candidates, selected_tables) if selected_tables else [] - updated = ensure_study_section(note_text, figures, tables) - args.note.write_text(updated, encoding="utf-8") - return 0 - - if args.command == "validate-selected": - note_text = args.note.read_text(encoding="utf-8") - fulltext = args.fulltext.read_text(encoding="utf-8") - figure_candidates = extract_figures_from_fulltext(fulltext) - table_candidates = extract_tables_from_fulltext(fulltext) - selected_figures = {part.strip() for part in (args.figures or "").split(",") if part.strip()} - selected_tables = {part.strip() for part in (args.tables or "").split(",") if part.strip()} - figures = select_entries_by_numbers(figure_candidates, selected_figures, attr="image_id") if selected_figures else [] - tables = select_entries_by_numbers(table_candidates, selected_tables) if selected_tables else [] - missing = validate_selected_blocks(note_text, figures, tables) - if missing: - for item in missing: - print(item) - return 1 - print("OK") - return 0 - - if args.command == "validate-note": - note_text = args.note.read_text(encoding="utf-8") - fulltext = args.fulltext.read_text(encoding="utf-8") - # Auto-load figure-map.json from same directory as fulltext if available - figure_map = None - figure_map_path = args.fulltext.parent / "figure-map.json" - if figure_map_path.exists(): - try: - figure_map = json.loads(figure_map_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, UnicodeDecodeError): - pass - figure_candidates = extract_figures_from_fulltext(fulltext, figure_map) - table_candidates = extract_tables_from_fulltext(fulltext) - selected_figures = {part.strip() for part in (args.figures or "").split(",") if part.strip()} - selected_tables = {part.strip() for part in (args.tables or "").split(",") if part.strip()} - figures = ( - select_entries_by_numbers(figure_candidates, selected_figures, attr="image_id") - if selected_figures - else build_figure_plan(figure_candidates) - ) - tables = select_entries_by_numbers(table_candidates, selected_tables) if selected_tables else [] - issues = validate_deep_note(note_text, figures, tables) - if issues: - for item in issues: - print(item) - return 1 - print("OK") - return 0 - - if args.command == "queue": - queue = scan_deep_reading_queue(args.vault) - if args.format == "json": - print(json.dumps(queue, ensure_ascii=False, indent=2)) - else: - ready = [row for row in queue if row["ocr_status"] == "done"] - blocked = [row for row in queue if row["ocr_status"] != "done"] - print(f"# 待精读队列 ({len(queue)} 篇)") - print() - if ready: - print(f"## 就绪 ({len(ready)} 篇) — OCR 完成") - print() - for row in ready: - print(f"- `{row['zotero_key']}` | {row['domain']} | {row['title']}") - print() - if blocked: - print(f"## 阻塞 ({len(blocked)} 篇) — 等待 OCR") - print() - for row in blocked: - print(f"- `{row['zotero_key']}` | {row['domain']} | {row['title']} | OCR: {row['ocr_status']}") - print() - if not ready and not blocked: - print("暂无待精读论文。") - return 0 - - if args.command == "figure-map": - fulltext = args.fulltext.read_text(encoding="utf-8") - result = build_figure_map(fulltext, zotero_key=args.key) - result["generated_at"] = __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() - if args.out: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") - print(f"figure-map: wrote {args.out}") - else: - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - if args.command == "chart-type-scan": - figure_map = _read_json(args.figure_map) - if not figure_map: - print("ERROR: Could not read figure-map.json or file is empty", file=__import__("sys").stderr) - return 1 - result = build_chart_type_map(figure_map) - if args.out: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") - print(f"chart-type-scan: wrote {args.out}") - else: - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - return 1 - - -if __name__ == "__main__": - raise SystemExit(main())