From e91dfd8c450cd6ba7ae42c58088a52ca57172a7c Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Wed, 13 May 2026 16:58:39 +0800 Subject: [PATCH] perf: refresh_paper accepts entry dict, skip full index read --- paperforge/memory/refresh.py | 43 +++++++++++++++---------------- paperforge/worker/asset_index.py | 29 +++++++++++++++------ tests/unit/memory/test_refresh.py | 10 ++++++- 3 files changed, 51 insertions(+), 31 deletions(-) diff --git a/paperforge/memory/refresh.py b/paperforge/memory/refresh.py index ebd033ab..1998d6a3 100644 --- a/paperforge/memory/refresh.py +++ b/paperforge/memory/refresh.py @@ -1,18 +1,17 @@ from __future__ import annotations import json +from datetime import datetime, timezone from pathlib import Path from paperforge.memory.builder import ( - PAPER_COLUMNS, - ASSET_FIELDS, ALIAS_TYPES, - compute_hash, + ASSET_FIELDS, + PAPER_COLUMNS, _resolve_vault_path, ) from paperforge.memory.db import get_connection, get_memory_db_path from paperforge.memory.schema import ensure_schema -from paperforge.worker.asset_index import read_index from paperforge.worker.asset_state import ( compute_lifecycle, compute_maturity, @@ -20,22 +19,13 @@ from paperforge.worker.asset_state import ( ) -def refresh_paper(vault: Path, zotero_key: str) -> bool: - """Incrementally refresh one paper in paperforge.db from formal-library.json.""" - envelope = read_index(vault) - if not envelope: - return False - items = envelope if isinstance(envelope, list) else envelope.get("items", []) - - entry = None - for e in items: - if e.get("zotero_key") == zotero_key: - entry = e - break - if not entry: +def refresh_paper(vault: Path, entry: dict) -> bool: + """Upsert a single paper into memory DB. Entry is from _build_entry() output.""" + zotero_key = entry.get("zotero_key", "") + if not zotero_key: return False - generated_at = envelope.get("generated_at", "") if not isinstance(envelope, list) else "" + generated_at = datetime.now(timezone.utc).isoformat() db_path = get_memory_db_path(vault) if not db_path.exists(): @@ -116,10 +106,19 @@ def refresh_paper(vault: Path, zotero_key: str) -> bool: conn.execute( "INSERT INTO paper_fts(rowid, zotero_key, citation_key, title, first_author, authors_json, abstract, journal, domain, collection_path, collections_json) " "VALUES ((SELECT rowid FROM papers WHERE zotero_key = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - (zotero_key, zotero_key, entry.get("citation_key", ""), entry.get("title", ""), - entry.get("first_author", ""), paper_values["authors_json"], - entry.get("abstract", ""), entry.get("journal", ""), entry.get("domain", ""), - entry.get("collection_path", ""), paper_values["collections_json"]), + ( + zotero_key, + zotero_key, + entry.get("citation_key", ""), + entry.get("title", ""), + entry.get("first_author", ""), + paper_values["authors_json"], + entry.get("abstract", ""), + entry.get("journal", ""), + entry.get("domain", ""), + entry.get("collection_path", ""), + paper_values["collections_json"], + ), ) except Exception: pass # FTS may not be available diff --git a/paperforge/worker/asset_index.py b/paperforge/worker/asset_index.py index ed9e7519..6bdbd99e 100644 --- a/paperforge/worker/asset_index.py +++ b/paperforge/worker/asset_index.py @@ -37,7 +37,6 @@ import filelock from paperforge import __version__ as _paperforge_version from paperforge.adapters.obsidian_frontmatter import ( _legacy_control_flags, - read_frontmatter_bool, read_frontmatter_dict, read_frontmatter_optional_bool, ) @@ -230,7 +229,10 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: Lazy imports inside avoid circular dependencies with ``sync.py``. """ # Lazy imports to avoid circular deps with sync.py - from paperforge.worker._utils import read_json, slugify_filename, write_json, yaml_quote + import shutil + + from paperforge import __version__ as PAPERFORGE_VERSION + from paperforge.worker._utils import lookup_impact_factor, read_json, slugify_filename, write_json, yaml_quote from paperforge.worker.asset_state import ( compute_health, compute_lifecycle, @@ -246,9 +248,6 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: obsidian_wikilink_for_path, obsidian_wikilink_for_pdf, ) - from paperforge.worker._utils import lookup_impact_factor - from paperforge import __version__ as PAPERFORGE_VERSION - import shutil key = item["key"] collection_meta = collection_fields(item.get("collections", [])) @@ -291,8 +290,8 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: if "aliases:" not in text[: text.find("\n---", 4)]: alias_line = f"aliases: [{yaml_quote(item.get('title', ''))}, {yaml_quote(item.get('citation_key') or item.get('key', ''))}]\n" text = re.sub( - r'(^title:.*\n)', - r'\1' + alias_line, + r"(^title:.*\n)", + r"\1" + alias_line, text, count=1, flags=re.MULTILINE, @@ -338,6 +337,7 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: return str(fm.get(key, "")).strip() except Exception: return "" + note_dr = _read_fm_str(main_note_path, "deep_reading_status") if not note_dr: note_dr = _read_fm_str(note_path, "deep_reading_status") @@ -415,7 +415,7 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: text = main_note_path.read_text(encoding="utf-8") fm_close = text.find("---\n", 4) # closing --- after opening --- if fm_close != -1: - body = text[fm_close + 4:] # everything after frontmatter + body = text[fm_close + 4 :] # everything after frontmatter new_full = frontmatter_note(entry, "") new_fm_close = new_full.find("---\n", 4) if new_fm_close != -1: @@ -492,6 +492,12 @@ def build_index(vault: Path, verbose: bool = False) -> int: for item in export_rows: entry = _build_entry(item, vault, paths, domain, zotero_dir) index_rows.append(entry) + try: + from paperforge.memory.refresh import refresh_paper + + refresh_paper(vault, entry) + except Exception: + pass # memory DB refresh is best-effort # Atomically write the envelope-wrapped index index_path = paths["index"] @@ -572,6 +578,13 @@ def refresh_index_entry(vault: Path, key: str) -> bool: # Build single entry and update the items list new_entry = _build_entry(found_item, vault, paths, found_domain, zotero_dir) + try: + from paperforge.memory.refresh import refresh_paper + + refresh_paper(vault, new_entry) + except Exception: + pass # memory DB refresh is best-effort + replaced = False for i, existing_entry in enumerate(items): if existing_entry.get("zotero_key") == key: diff --git a/tests/unit/memory/test_refresh.py b/tests/unit/memory/test_refresh.py index 4f1bd2e6..103440fa 100644 --- a/tests/unit/memory/test_refresh.py +++ b/tests/unit/memory/test_refresh.py @@ -6,4 +6,12 @@ from paperforge.memory.refresh import refresh_paper def test_refresh_paper_returns_false_when_no_db(): - assert refresh_paper(Path("/nonexistent/vault"), "KEY001") is False + assert refresh_paper(Path("/nonexistent/vault"), {"zotero_key": "KEY001"}) is False + + +def test_refresh_paper_returns_false_for_empty_key(): + assert refresh_paper(Path("/nonexistent/vault"), {}) is False + + +def test_refresh_paper_returns_false_for_missing_key(): + assert refresh_paper(Path("/nonexistent/vault"), {"title": "No Key"}) is False