obsidian-plugin/lllin000_PaperForge
1
0
Fork 0
mirror of https://github.com/lllin000/PaperForge.git synced 2026-07-22 17:00:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: executemany batching in builder + file read consolidation

Browse source
This commit is contained in:
Research Assistant 2026-05-13 17:17:08 +08:00
parent c380f205e3
commit c8cd9f5578
2 changed files with 54 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
paperforge/memory/builder.py
View file

@ -3,7 +3,6 @@ from __future__ import annotations
import hashlib
import json
import logging
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
@ -89,9 +88,13 @@ def build_from_index(vault: Path) -> dict:
conn.execute("DROP TRIGGER IF EXISTS papers_ai")
now_utc = datetime.now(timezone.utc).isoformat()
papers_count = 0
assets_count = 0
aliases_count = 0
paper_rows: list[dict] = []
asset_rows: list[tuple] = []
alias_rows: list[tuple] = []
placeholders = ", ".join([f":{c}" for c in PAPER_COLUMNS])
cols = ", ".join(PAPER_COLUMNS)
paper_sql = f"INSERT OR REPLACE INTO papers ({cols}) VALUES ({placeholders})"
for entry in items:
zotero_key = entry.get("zotero_key", "")
@ -101,15 +104,7 @@ def build_from_index(vault: Path) -> dict:
entry["lifecycle"] = str(compute_lifecycle(entry))
entry["maturity"] = compute_maturity(entry)
entry["next_step"] = str(compute_next_step(entry))
paper_values = build_paper_row(entry, generated_at)
placeholders = ", ".join([f":{c}" for c in PAPER_COLUMNS])
cols = ", ".join(PAPER_COLUMNS)
conn.execute(
f"INSERT OR REPLACE INTO papers ({cols}) VALUES ({placeholders})",
paper_values,
)
papers_count += 1
paper_rows.append(build_paper_row(entry, generated_at))
for asset_type, entry_field in ASSET_FIELDS:
path_val = entry.get(entry_field, "")
@ -126,31 +121,28 @@ def build_from_index(vault: Path) -> dict:
except Exception:
exists = 0
conn.execute(
"""INSERT OR REPLACE INTO paper_assets
(paper_id, asset_type, path, exists_on_disk)
VALUES (?, ?, ?, ?)""",
(zotero_key, asset_type, rel_path, exists),
)
assets_count += 1
asset_rows.append((zotero_key, asset_type, rel_path, exists))
for alias_type in ALIAS_TYPES:
raw_val = entry.get(alias_type, "")
if not raw_val:
continue
raw_str = str(raw_val)
conn.execute(
"""INSERT OR REPLACE INTO paper_aliases
(paper_id, alias, alias_norm, alias_type)
VALUES (?, ?, ?, ?)""",
(
zotero_key,
raw_str,
raw_str.lower().strip(),
alias_type,
),
)
aliases_count += 1
alias_rows.append((zotero_key, raw_str, raw_str.lower().strip(), alias_type))
conn.executemany(paper_sql, paper_rows)
conn.executemany(
"""INSERT OR REPLACE INTO paper_assets
(paper_id, asset_type, path, exists_on_disk)
VALUES (?, ?, ?, ?)""",
asset_rows,
)
conn.executemany(
"""INSERT OR REPLACE INTO paper_aliases
(paper_id, alias, alias_norm, alias_type)
VALUES (?, ?, ?, ?)""",
alias_rows,
)
conn.execute("""INSERT INTO paper_fts(rowid, zotero_key, citation_key, title, first_author, authors_json, abstract, journal, domain, collection_path, collections_json)
SELECT rowid, zotero_key, citation_key, title, first_author, authors_json, abstract, journal, domain, collection_path, collections_json
@ -175,9 +167,9 @@ def build_from_index(vault: Path) -> dict:
return {
"db_path": str(db_path),
"papers_indexed": papers_count,
"assets_indexed": assets_count,
"aliases_indexed": aliases_count,
"papers_indexed": len(paper_rows),
"assets_indexed": len(asset_rows),
"aliases_indexed": len(alias_rows),
"schema_version": str(CURRENT_SCHEMA_VERSION),
}
except Exception:

44
paperforge/worker/asset_index.py
View file

@ -299,7 +299,6 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
except Exception:
pass # alias will be injected on next full frontmatter_note pass
break # only one old file per key
deep_reading_file = workspace_dir / "deep-reading.md"
target_fulltext = workspace_dir / "fulltext.md"
source_fulltext = paths["ocr"] / key / "fulltext.md"
@ -310,7 +309,6 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
logger.info("Bridged fulltext.md to workspace for %s", key)
fulltext_exists = target_fulltext.exists()
deep_reading_exists = deep_reading_file.exists()
# ---- entry dict -------------------------------------------------------
authors = item.get("authors", [])
@ -320,12 +318,17 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
legacy_flags = _legacy_control_flags(paths, key)
legacy_do_ocr = legacy_flags.get("do_ocr")
legacy_analyze = legacy_flags.get("analyze")
# Single frontmatter read for all control fields
# Single frontmatter read for all control fields — cache text for reuse
fm = {}
fm_cached_text = ""
fm_was_main = False
for fp in (main_note_path, note_path):
if fp and fp.exists():
try:
fm = read_frontmatter_dict(fp.read_text(encoding="utf-8"))
note_text = fp.read_text(encoding="utf-8")
fm = read_frontmatter_dict(note_text)
fm_cached_text = note_text
fm_was_main = (fp == main_note_path)
break
except Exception:
continue
@ -344,6 +347,19 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
if analyze_value is None:
analyze_value = meta.get("analyze") is True or meta.get("deep_reading_status") == "done"
# Compute deep reading status once, reusing cached text when possible.
# main_note_path is canonical — don't fall back to note_path when it exists.
_dr_status = "pending"
if note_dr == "done":
_dr_status = "done"
elif fm_was_main:
_dr_status = "done" if has_deep_reading_content(fm_cached_text) else "pending"
elif main_note_path.exists():
try:
_dr_status = "done" if has_deep_reading_content(main_note_path.read_text(encoding="utf-8")) else "pending"
except Exception:
pass
entry = {
"zotero_key": key,
"citation_key": item.get("citation_key", ""),
@ -371,23 +387,15 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
"ocr_job_id": meta.get("ocr_job_id", ""),
"ocr_md_path": obsidian_wikilink_for_path(vault, meta.get("markdown_path", "")),
"ocr_json_path": meta.get("json_path", ""),
"deep_reading_status": (
"done"
if note_dr == "done"
else "done"
if main_note_path.exists() and has_deep_reading_content(main_note_path.read_text(encoding="utf-8"))
else "done"
if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding="utf-8"))
else "pending"
),
"deep_reading_status": _dr_status,
"note_path": str((main_note_path if main_note_path.exists() else note_path).relative_to(vault)).replace(
"\\", "/"
),
"deep_reading_md_path": (
str(main_note_path.relative_to(vault)).replace("\\", "/")
if main_note_path.exists() and has_deep_reading_content(main_note_path.read_text(encoding="utf-8"))
if _dr_status == "done" and main_note_path.exists()
else str(note_path.relative_to(vault)).replace("\\", "/")
if note_path.exists() and has_deep_reading_content(note_path.read_text(encoding="utf-8"))
if _dr_status == "done" and note_path.exists()
else ""
),
# Workspace path fields are only advertised when the backing files/dirs exist.
@ -406,7 +414,7 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
# Slug already frozen above — for existing notes, update frontmatter only (preserve body)
if main_note_path.exists():
text = main_note_path.read_text(encoding="utf-8")
text = fm_cached_text if fm_was_main else main_note_path.read_text(encoding="utf-8")
fm_close = text.find("---\n", 4) # closing --- after opening ---
if fm_close != -1:
body = text[fm_close + 4 :] # everything after frontmatter
@ -420,7 +428,9 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir:
else:
main_note_path.write_text(frontmatter_note(entry, text), encoding="utf-8")
else:
existing_text = note_path.read_text(encoding="utf-8") if note_path.exists() else ""
existing_text = fm_cached_text if not fm_was_main and fm_cached_text else (
note_path.read_text(encoding="utf-8") if note_path.exists() else ""
)
main_note_path.write_text(frontmatter_note(entry, existing_text), encoding="utf-8")
# Write per-workspace paper-meta.json (Phase 37: internal state outside frontmatter)