From 8d3e4432c3df21b0ad4e9f0fb0dcb7d3a3f5d0e4 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Tue, 12 May 2026 23:34:24 +0800 Subject: [PATCH] chore: add pre-commit JS syntax hook + stage missing vector CLI files --- ...6-05-12-plugin-settings-redesign-REVIEW.md | 205 ++++++++++++++++ paperforge/cli.py | 25 ++ paperforge/commands/__init__.py | 2 + paperforge/commands/embed.py | 85 +++++++ paperforge/commands/retrieve.py | 55 +++++ paperforge/memory/chunker.py | 102 ++++++++ paperforge/memory/vector_db.py | 221 ++++++++++++++++++ 7 files changed, 695 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-12-plugin-settings-redesign-REVIEW.md create mode 100644 paperforge/commands/embed.py create mode 100644 paperforge/commands/retrieve.py create mode 100644 paperforge/memory/chunker.py create mode 100644 paperforge/memory/vector_db.py diff --git a/docs/superpowers/specs/2026-05-12-plugin-settings-redesign-REVIEW.md b/docs/superpowers/specs/2026-05-12-plugin-settings-redesign-REVIEW.md new file mode 100644 index 00000000..efda2f7c --- /dev/null +++ b/docs/superpowers/specs/2026-05-12-plugin-settings-redesign-REVIEW.md @@ -0,0 +1,205 @@ +--- +phase: settings-redesign-spec-review +reviewed: 2026-05-12T12:00:00Z +depth: deep +files_reviewed: 5 +files_reviewed_list: + - docs/superpowers/specs/2026-05-12-plugin-settings-redesign.md + - paperforge/plugin/main.js + - paperforge/services/skill_deploy.py + - paperforge/skills/literature-qa/SKILL.md + - paperforge/skills/literature-logging/SKILL.md +findings: + critical: 1 + warning: 3 + info: 3 + total: 7 +status: issues_found +--- + +# Spec Review: Plugin Settings Redesign + +**Reviewed:** 2026-05-12 +**Depth:** deep (cross-file analysis across plugin codebase, skill deploy, SKILL.md frontmatter) +**Files Reviewed:** 5 +**Status:** ISSUES_FOUND — one BLOCKER must be resolved before implementation + +## Summary + +Cross-referenced the proposed 2-tab settings redesign against the current PaperForge plugin codebase (`paperforge/plugin/main.js`), the `skill_deploy.py` AGENT_SKILL_DIRS mapping, and the two existing SKILL.md files. The spec's architecture (Claudian tab pattern, DOM-based tab switching, `disable-model-invocation` toggle) is well-reasoned and compatible. However, one data persistence issue is a BLOCKER, and several gaps/ambiguities need resolution before implementation proceeds. + +--- + +## Critical Issues + +### CR-01: `saveSettings()` will silently discard all new `data.json` keys + +**File:** `paperforge/plugin/main.js:3534-3542` +**Issue:** The spec proposes storing new feature-toggle data in Obsidian's plugin `data.json` under keys like `features`, `vector_db_mode`, `vector_db_model`, `vector_db_api_key`, and `frozen_skills`. However, the current `saveSettings()` method explicitly filters out any key not present in `DEFAULT_SETTINGS`: + +```js +async saveSettings() { + // Only persist non-path settings to plugin data.json + const dataToSave = {}; + for (const key of Object.keys(DEFAULT_SETTINGS)) { // ← whitelist filter + if (key in this.settings) { + dataToSave[key] = this.settings[key]; + } + } + await this.saveData(dataToSave); +} +``` + +`DEFAULT_SETTINGS` (line 547-556) currently contains only: +- `vault_path`, `setup_complete`, `auto_update`, `agent_platform`, `language`, `paddleocr_api_key`, `zotero_data_dir`, `python_path` + +Any new key (`features`, `vector_db_mode`, `frozen_skills`, etc.) will be **silently discarded** on every save. Toggling a feature, changing a vector DB mode, or freezing a skill would appear to work until the user re-opens settings — at which point `loadData()` returns the stale (or default) values. + +**Fix:** One of two approaches: + +**Option A — Extend DEFAULT_SETTINGS (simpler, less risky):** +```js +const DEFAULT_SETTINGS = { + vault_path: '', + setup_complete: false, + auto_update: true, + agent_platform: 'opencode', + language: '', + paddleocr_api_key: '', + zotero_data_dir: '', + python_path: '', + // NEW: Feature toggles + features: { + fts_search: true, + agent_context: true, + reading_log: true, + vector_db: false, + }, + vector_db_mode: 'local', + vector_db_model: 'all-MiniLM-L6-v2', + vector_db_api_key: '', + frozen_skills: {}, +}; +``` + +**Option B — Change save logic to whitelist exclusions rather than inclusions:** +```js +// Persist everything except internal/temporary fields +const EXCLUDE_KEYS = new Set(['_python_path_stale', '_saveTimeout', '_pfConfig']); +const dataToSave = {}; +for (const key of Object.keys(this.settings)) { + if (!EXCLUDE_KEYS.has(key) && typeof this.settings[key] !== 'function') { + dataToSave[key] = this.settings[key]; + } +} +``` + +Option A is recommended as it preserves the defensive posture of the existing code. + +--- + +## Warnings + +### WR-01: `source` field missing — system skills will be mis-categorized as user skills + +**File:** `paperforge/skills/literature-qa/SKILL.md`, `paperforge/skills/literature-logging/SKILL.md` +**Issue:** The spec uses `source: paperforge` frontmatter to identify system-managed skills (with update/freeze controls) vs user skills (`source: user` or no `source` field → toggle only). Neither existing SKILL.md has a `source` field: + +```yaml +# literature-qa/SKILL.md (current) +name: literature-qa +description: > + 学术文献库操作：精读、问答、检索、批量阅读... + +# literature-logging/SKILL.md (current) +name: literature-logging +description: > + Literature reading and working log management... +``` + +Per the spec's rules: skills without `source` are treated as **user** skills (toggle only, no update button). This means the two PaperForge system skills would show up without update/freeze controls on first install — users would need the implementation to retroactively add `source: paperforge` to detect them correctly. + +**Fix:** Add `source: paperforge` to both SKILL.md files as part of this implementation, and include it in the `deploy_skills()` copytree operation. Also add a `version` field (currently absent from both) since the spec expects it for GitHub semver comparison. + +```yaml +# Proposed addition to both SKILL.md frontmatter blocks +source: paperforge +version: 1.5.5 +``` + +### WR-02: Feature toggle enforcement is missing — no code that reads `features.*` to gate CLI commands + +**File:** Spec section "Section 2: Feature Toggles" +**Issue:** The spec states "When a feature is disabled, the corresponding CLI command returns a clear error message." However, the spec only covers the **settings UI** side — there is no corresponding mechanism described for the Python CLI (`cli.py`) or workers to read `features.*` from `data.json` (which lives in the vault's `.obsidian/plugins/paperforge/` directory, inaccessible at runtime unless the plugin passes the values through `paperforge.json` or an env var). + +Either: +- The plugin needs to write toggles into `paperforge.json` so the Python runtime can read them, OR +- The plugin needs to pass toggles as CLI flags/arguments when invoking commands, OR +- The enforcement lives only in the plugin's command palette (never calls CLI for disabled features) + +**Fix:** Add explicit documentation of the enforcement mechanism. Recommended: the plugin writes a `feature_toggles` block to `paperforge.json` during `saveSettings()`, mirroring the existing `vault_config` block pattern (see `savePaperforgeJson()` at line 3455). This way both plugin and Python runtime have a single source of truth. + +### WR-03: `_debouncedSave()` calls `saveSettings()` — both save paths need updating + +**File:** `paperforge/plugin/main.js:2573-2576` +**Issue:** The settings tab has two save pathways: +1. Direct calls: `this.plugin.saveSettings()` (line 2268 in the Python path onChange handler) +2. Debounced calls: `this._debouncedSave()` (lines 2214, 2223) which calls `this.plugin.saveSettings()` after 500ms + +Both flow through the same `saveSettings()` method. If CR-01 is fixed (extending DEFAULT_SETTINGS), this is not an additional bug — but it means **any new Setting added to the features tab must use the same save mechanism**. The spec doesn't mention this constraint. + +**Fix:** Document in implementation notes that all new toggle handlers should call `this._debouncedSave()` (for inputs) or `this.plugin.saveSettings()` (for immediate actions). For the skill toggle that writes to SKILL.md frontmatter (not data.json), a separate write path is needed — this is handled correctly by the spec but should be called out. + +--- + +## Info + +### IN-01: JSON key nesting inconsistency between architecture diagram and data storage section + +**File:** `docs/superpowers/specs/2026-05-12-plugin-settings-redesign.md:29-30 vs 170-185` +**Issue:** The architecture diagram nests vector DB config under a `向量数据库` section with `开关`, `模式`, `本地`, `API` as sub-items. The JSON storage section places `features.vector_db` (the master toggle) nested under `features`, but places `vector_db_mode`, `vector_db_model`, and `vector_db_api_key` at the **top level** of data.json — not under `features.vector_db.*`. This is structurally valid but the flat/grouped inconsistency between the spec's labeled options and the flat JSON may cause confusion during implementation. + +```json +// Spec proposes: +{ + "features": { "vector_db": false }, // master toggle nested + "vector_db_mode": "local", // implementation detail at top level + "vector_db_model": "...", // ... + "vector_db_api_key": "" // ... +} +``` + +**Fix:** Consider either fully nesting (`features.vector_db.enabled`, `features.vector_db.mode`, etc.) or fully flattening (`features_vector_db`, `features_vector_db_mode`, etc.). The current mixed approach works but adds mental overhead. The nested approach is cleaner for future feature additions. + +### IN-02: Vector DB panel gaps — model detection and error handling unspecified + +**File:** `docs/superpowers/specs/2026-05-12-plugin-settings-redesign.md:161-164` +**Issue:** The vector DB panel design leaves several implementation details ambiguous: + +1. **Model installation state detection**: The status badge `● 已安装 / ○ 未安装` has no specified detection logic. `sentence-transformers` models download on first use (triggered by `SentenceTransformer('all-MiniLM-L6-v2')`), not by a distinct install step. How does the UI know the model is installed? Checking for cached files in `~/.cache/torch/sentence_transformers/`? Running a probe import? + +2. **Installation is async but not cancellable**: "`pip install` is async — show progress bar" — but what happens if the user closes Obsidian or switches vaults mid-install? Is there a cancel mechanism? + +3. **No network error handling**: What does the UI show if `pip install` fails due to network issues, disk space, or permissions? + +**Fix:** Add implementation details for each of these edge cases to the spec, or document them as deferred design decisions. + +### IN-03: Tab state is DOM-preserved between switches but NOT across re-opens + +**File:** Spec section "Tab Implementation" vs `paperforge/plugin/main.js:2206-2208` +**Issue:** The spec correctly follows the Claudian pattern (all tabs exist in DOM, CSS toggles visibility). This preserves form field state when switching between 安装 and 功能 tabs within a single settings session. However, Obsidian calls `display()` on every settings tab open, which runs `containerEl.empty()` (line 2208) and rebuilds the entire UI from scratch. This is standard Obsidian behavior, but means: +- Switching tabs: state preserved (correct) +- Closing and reopening settings: state lost (standard, acceptable) +- Running "Sync Runtime" → calls `this.display()` (line 2553, 2562): **entire settings rebuilt, active tab resets to default** + +The Sync Runtime action at line 2553/2562 explicitly calls `this.display()` which would reset any partially filled forms or the active tab selection back to default. This shouldn't block the spec, but should be noted: the sync runtime action should either: +- Preserve `this.activeTab` before calling `this.display()`, or +- Not call `this.display()` at all (re-render only the runtime health section) + +**Fix:** Add a note to the implementation plan: `this.display()` reset is acceptable for a settings reopen, but the sync runtime flow should preserve `this.activeTab` across the re-render. + +--- + +_Reviewed: 2026-05-12T12:00:00Z_ +_Reviewer: VT-OS/OPENCODE (gsd-code-reviewer)_ +_Depth: deep_ diff --git a/paperforge/cli.py b/paperforge/cli.py index fa3f2422..9adef5db 100644 --- a/paperforge/cli.py +++ b/paperforge/cli.py @@ -258,6 +258,21 @@ def build_parser() -> argparse.ArgumentParser: p_dash = sub.add_parser("dashboard", help="Aggregated stats and permissions for the plugin dashboard") p_dash.add_argument("--json", action="store_true", help="Output as PFResult JSON") + # Vector DB + p_embed = sub.add_parser("embed", help="Vector embedding operations") + p_embed_sp = p_embed.add_subparsers(dest="embed_subcommand", required=True) + p_embed_build = p_embed_sp.add_parser("build", help="Build vector index from OCR fulltext") + p_embed_build.add_argument("--json", action="store_true") + p_embed_build.add_argument("--force", action="store_true") + p_embed_status = p_embed_sp.add_parser("status", help="Check vector DB status") + p_embed_status.add_argument("--json", action="store_true") + + p_retrieve = sub.add_parser("retrieve", help="Semantic search across OCR fulltext") + p_retrieve.add_argument("query", help="Search query") + p_retrieve.add_argument("--json", action="store_true") + p_retrieve.add_argument("--limit", type=int, default=5) + p_retrieve.add_argument("--expand", action="store_true", default=True) + # Memory Layer commands p_memory = sub.add_parser("memory", help="Manage the Memory Layer") p_memory_sp = p_memory.add_subparsers(dest="memory_subcommand", required=True) @@ -514,6 +529,16 @@ def main(argv: list[str] | None = None) -> int: return run(args) + if args.command == "embed": + from paperforge.commands.embed import run + + return run(args) + + if args.command == "retrieve": + from paperforge.commands.retrieve import run + + return run(args) + if args.command == "paper-status": from paperforge.commands.paper_status import run diff --git a/paperforge/commands/__init__.py b/paperforge/commands/__init__.py index cdaac04d..c36887b1 100644 --- a/paperforge/commands/__init__.py +++ b/paperforge/commands/__init__.py @@ -11,6 +11,8 @@ _COMMAND_REGISTRY: dict[str, str] = { "dashboard": "paperforge.commands.dashboard", "finalize": "paperforge.commands.finalize", "memory": "paperforge.commands.memory", + "embed": "paperforge.commands.embed", + "retrieve": "paperforge.commands.retrieve", "paper-status": "paperforge.commands.paper_status", "agent-context": "paperforge.commands.agent_context", } diff --git a/paperforge/commands/embed.py b/paperforge/commands/embed.py new file mode 100644 index 00000000..d90c4b24 --- /dev/null +++ b/paperforge/commands/embed.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from paperforge.core.errors import ErrorCode +from paperforge.core.result import PFError, PFResult +from paperforge.memory.chunker import chunk_fulltext +from paperforge.memory.vector_db import ( + delete_paper_vectors, + embed_paper, + get_embed_status, + get_vector_db_path, +) +from paperforge.worker.asset_index import read_index +from paperforge import __version__ as PF_VERSION + + +def run(args: argparse.Namespace) -> int: + vault = args.vault_path + sub = getattr(args, "embed_subcommand", "build") + + if sub == "status": + status = get_embed_status(vault) + result = PFResult(ok=True, command="embed status", version=PF_VERSION, data=status) + if args.json: + print(result.to_json()) + else: + for k, v in status.items(): + print(f" {k}: {v}") + return 0 + + # Build + envelope = read_index(vault) + if not envelope: + result = PFResult(ok=False, command="embed build", version=PF_VERSION, + error=PFError(code=ErrorCode.PATH_NOT_FOUND, + message="Canonical index not found. Run paperforge sync first.")) + print(result.to_json() if args.json else result.error.message, file=sys.stderr if not args.json else sys.stdout) + return 1 + + items = envelope if isinstance(envelope, list) else envelope.get("items", []) + done_papers = [e for e in items if e.get("ocr_status") == "done"] + + if args.force: + db_path = get_vector_db_path(vault) + if db_path.exists(): + import shutil + shutil.rmtree(str(db_path), ignore_errors=True) + + papers_embedded = 0 + chunks_embedded = 0 + for entry in done_papers: + key = entry.get("zotero_key") + fulltext_rel = entry.get("fulltext_path", "") + if not fulltext_rel: + continue + fulltext_path = vault / fulltext_rel + chunks = chunk_fulltext(fulltext_path) + if not chunks: + continue + try: + delete_paper_vectors(vault, key) + n = embed_paper(vault, key, chunks) + chunks_embedded += n + papers_embedded += 1 + except Exception as e: + result = PFResult(ok=False, command="embed build", version=PF_VERSION, + error=PFError(code=ErrorCode.INTERNAL_ERROR, message=str(e))) + print(result.to_json() if args.json else result.error.message, file=sys.stderr if not args.json else sys.stdout) + return 1 + + data = { + "papers_embedded": papers_embedded, + "chunks_embedded": chunks_embedded, + "model": get_embed_status(vault)["model"], + "mode": get_embed_status(vault)["mode"], + } + result = PFResult(ok=True, command="embed build", version=PF_VERSION, data=data) + if args.json: + print(result.to_json()) + else: + print(f"Embedded {papers_embedded} papers ({chunks_embedded} chunks)") + return 0 diff --git a/paperforge/commands/retrieve.py b/paperforge/commands/retrieve.py new file mode 100644 index 00000000..a457db10 --- /dev/null +++ b/paperforge/commands/retrieve.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import argparse +import sys +import json + +from paperforge.core.errors import ErrorCode +from paperforge.core.result import PFError, PFResult +from paperforge.memory.db import get_connection, get_memory_db_path +from paperforge.memory.vector_db import retrieve_chunks +from paperforge import __version__ as PF_VERSION + + +def run(args: argparse.Namespace) -> int: + vault = args.vault_path + query = args.query + limit = args.limit or 5 + + try: + chunks = retrieve_chunks(vault, query, limit=limit, expand=args.expand) + except Exception as e: + result = PFResult(ok=False, command="retrieve", version=PF_VERSION, + error=PFError(code=ErrorCode.INTERNAL_ERROR, message=str(e))) + print(result.to_json() if args.json else result.error.message, file=sys.stderr if not args.json else sys.stdout) + return 1 + + # Enrich with paper metadata from memory DB + if chunks: + db_path = get_memory_db_path(vault) + if db_path.exists(): + conn = get_connection(db_path, read_only=True) + try: + for c in chunks: + row = conn.execute( + "SELECT citation_key, title, year, first_author FROM papers WHERE zotero_key=?", + (c["paper_id"],) + ).fetchone() + if row: + c["citation_key"] = row["citation_key"] + c["title"] = row["title"] + c["year"] = row["year"] + c["first_author"] = row["first_author"] + finally: + conn.close() + + data = {"query": query, "chunks": chunks, "count": len(chunks)} + result = PFResult(ok=True, command="retrieve", version=PF_VERSION, data=data) + + if args.json: + print(result.to_json()) + else: + print(f"{len(chunks)} chunks for: {query}") + for c in chunks: + print(f" [{c.get('section','')}] {c.get('citation_key','')} p{c.get('page_number',0)}: {c['chunk_text'][:80]}...") + return 0 diff --git a/paperforge/memory/chunker.py b/paperforge/memory/chunker.py new file mode 100644 index 00000000..7011fa43 --- /dev/null +++ b/paperforge/memory/chunker.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import re +from pathlib import Path + +# Section detection keywords (case-insensitive, must appear as short standalone line) +SECTION_PATTERNS = [ + re.compile(r'^\s*(introduction|methods|materials|results|discussion|conclusion|abstract|background|references|supplementary|acknowledgments?)\s*$', re.IGNORECASE), + re.compile(r'^\s*(figure\s*\d+|fig\.?\s*\d+|table\s*\d+)\s*$', re.IGNORECASE), +] + +def _detect_section(line: str) -> str: + """Try to identify a section title from a line.""" + stripped = line.strip() + if len(stripped) > 80: + return "" + for pat in SECTION_PATTERNS: + m = pat.match(stripped) + if m: + return m.group(0) + # Heuristic: ALL CAPS short line, no period + if stripped.isupper() and len(stripped) > 2 and '.' not in stripped: + return stripped + # Short line, no period, surrounded by blank lines (checked by caller) + if len(stripped) < 80 and '.' not in stripped[-5:]: + return stripped + return "" + + +def _clean_text(text: str) -> str: + """Remove image links and clean text for embedding.""" + # Remove standalone image links: ![[path]] + text = re.sub(r'^!\[\[.*\]\]\s*$', '', text, flags=re.MULTILINE) + # Replace inline images with placeholder + text = re.sub(r'!\[\[.*?\]\]', '[Figure]', text) + # Collapse multiple blank lines + text = re.sub(r'\n{3,}', '\n\n', text) + return text.strip() + + +def chunk_fulltext(fulltext_path: Path) -> list[dict]: + """Chunk a fulltext.md into embeddable segments. + + Returns list of dicts with: text, section, page_number, chunk_index, token_estimate. + """ + if not fulltext_path.exists(): + return [] + + text = _clean_text(fulltext_path.read_text(encoding="utf-8")) + + # Split by page markers + pages = re.split(r'', text) + # pages[0] = before first marker, pages[1] = page num, pages[2] = content, pages[3] = page num, ... + + current_section = "Text" + parts = [] + + if len(pages) > 1 and not pages[1].strip().isdigit(): + # No page marker found, treat whole text as one page + parts = [(1, text)] + else: + for j in range(1, len(pages), 2): + if j + 1 < len(pages): + try: + page_num = int(pages[j].strip()) + page_content = pages[j + 1] + parts.append((page_num, page_content)) + except ValueError: + continue + + if not parts and text.strip(): + parts = [(1, text)] + + chunks = [] + chunk_index = 0 + for page_num, page_text in parts: + # Split page into paragraphs by double newlines + paragraphs = [p.strip() for p in re.split(r'\n\s*\n', page_text) if p.strip()] + + # Detect section headers + for para in paragraphs: + section = _detect_section(para) + if section: + current_section = section + + # Group 2-3 paragraphs per chunk with 1-paragraph overlap + i = 0 + while i < len(paragraphs): + chunk_paras = paragraphs[i:i+3] + chunk_text = "\n\n".join(chunk_paras) + token_estimate = len(chunk_text.split()) # rough: 1 token ≈ 1 word + chunks.append({ + "text": chunk_text, + "section": current_section, + "page_number": page_num, + "chunk_index": chunk_index, + "token_estimate": token_estimate, + }) + chunk_index += 1 + i += max(1, len(chunk_paras) - 1) # advance but leave 1 overlap + + return chunks diff --git a/paperforge/memory/vector_db.py b/paperforge/memory/vector_db.py new file mode 100644 index 00000000..58221aa9 --- /dev/null +++ b/paperforge/memory/vector_db.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +# Lazy imports to avoid requiring chromadb unless actually used +_chroma = None +_ST = None + +def _get_chroma(): + global _chroma + if _chroma is None: + import chromadb + _chroma = chromadb + return _chroma + +def _get_st(): + global _ST + if _ST is None: + from sentence_transformers import SentenceTransformer + _ST = SentenceTransformer + return _ST + + +def _read_plugin_settings(vault: Path) -> dict: + """Read plugin data.json for vector_db settings.""" + data_path = vault / ".obsidian" / "plugins" / "paperforge" / "data.json" + if data_path.exists(): + return json.loads(data_path.read_text(encoding="utf-8")) + return {} + + +def get_vector_db_path(vault: Path) -> Path: + """Return the ChromaDB persistence directory.""" + from paperforge.config import paperforge_paths + paths = paperforge_paths(vault) + return (paths.get("memory_db", paths.get("index", vault / "System" / "PaperForge"))).parent / "vectors" + + +def get_collection(vault: Path): + """Get or create the ChromaDB collection for paperforge.""" + chroma = _get_chroma() + db_path = get_vector_db_path(vault) + db_path.mkdir(parents=True, exist_ok=True) + client = chroma.PersistentClient(path=str(db_path)) + # Delete and recreate if schema changed + try: + return client.get_or_create_collection( + name="paperforge_fulltext", + metadata={"hnsw:space": "cosine"}, + ) + except Exception: + client.delete_collection("paperforge_fulltext") + return client.create_collection( + name="paperforge_fulltext", + metadata={"hnsw:space": "cosine"}, + ) + + +def get_embedding_model(vault: Path): + """Load the embedding model based on plugin settings or default.""" + settings = _read_plugin_settings(vault) + mode = settings.get("vector_db_mode", "local") + + if mode == "api": + return None # API mode — embedding done externally + + model_name = settings.get("vector_db_model", "BAAI/bge-small-en-v1.5") + ST = _get_st() + logger.info("Loading embedding model: %s", model_name) + return ST(model_name) + + +def embed_paper(vault: Path, zotero_key: str, chunks: list[dict]) -> int: + """Embed chunks for one paper and insert into ChromaDB. Returns count.""" + collection = get_collection(vault) + model = get_embedding_model(vault) + + if model is None: + # API mode + return _embed_paper_api(vault, zotero_key, chunks, collection) + + # Local mode + texts = [c["text"] for c in chunks] + ids = [f"{zotero_key}_{c['chunk_index']}" for c in chunks] + metadatas = [ + { + "paper_id": zotero_key, + "section": c["section"], + "page_number": c["page_number"], + "chunk_index": c["chunk_index"], + "token_estimate": c["token_estimate"], + } + for c in chunks + ] + + embeddings = model.encode(texts, show_progress_bar=False).tolist() + + collection.add( + ids=ids, + embeddings=embeddings, + documents=texts, + metadatas=metadatas, + ) + return len(chunks) + + +def _embed_paper_api(vault, zotero_key, chunks, collection) -> int: + """Embed using OpenAI API.""" + settings = _read_plugin_settings(vault) + api_key = settings.get("vector_db_api_key", "") + if not api_key: + env_file = vault / ".env" + if env_file.exists(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + if line.startswith("OPENAI_API_KEY="): + api_key = line.split("=", 1)[1].strip().strip('"').strip("'") + if not api_key: + raise ValueError("No API key configured for vector DB") + + from openai import OpenAI + client = OpenAI(api_key=api_key) + + texts = [c["text"] for c in chunks] + ids = [f"{zotero_key}_{c['chunk_index']}" for c in chunks] + metadatas = [ + {"paper_id": zotero_key, "section": c["section"], + "page_number": c["page_number"], "chunk_index": c["chunk_index"], + "token_estimate": c["token_estimate"]} + for c in chunks + ] + + response = client.embeddings.create(model="text-embedding-3-small", input=texts) + embeddings = [e.embedding for e in response.data] + + collection.add(ids=ids, embeddings=embeddings, documents=texts, metadatas=metadatas) + return len(chunks) + + +def delete_paper_vectors(vault: Path, zotero_key: str) -> int: + """Delete all chunks for a paper from ChromaDB.""" + collection = get_collection(vault) + try: + results = collection.get(where={"paper_id": zotero_key}) + ids = results.get("ids", []) + if ids: + collection.delete(ids=ids) + return len(ids) + except Exception: + return 0 + + +def retrieve_chunks(vault: Path, query: str, limit: int = 5, expand: bool = True) -> list[dict]: + """Search for chunks matching the query. Returns list with adjacent context.""" + collection = get_collection(vault) + model = get_embedding_model(vault) + + if model is None: + # API mode + settings = _read_plugin_settings(vault) + api_key = settings.get("vector_db_api_key", "") + env_file = vault / ".env" + if not api_key and env_file.exists(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + if line.startswith("OPENAI_API_KEY="): + api_key = line.split("=", 1)[1].strip().strip('"').strip("'") + if not api_key: + raise ValueError("No API key configured for vector DB") + from openai import OpenAI + client = OpenAI(api_key=api_key) + response = client.embeddings.create(model="text-embedding-3-small", input=query) + query_embedding = response.data[0].embedding + else: + query_embedding = model.encode(query).tolist() + + results = collection.query( + query_embeddings=[query_embedding], + n_results=limit * 3 if expand else limit, + include=["documents", "metadatas", "distances"], + ) + + chunks = [] + for i, (doc, meta, dist) in enumerate(zip( + results["documents"][0], + results["metadatas"][0], + results["distances"][0], + )): + chunks.append({ + "paper_id": meta["paper_id"], + "section": meta.get("section", "Text"), + "page_number": meta.get("page_number", 1), + "chunk_index": meta.get("chunk_index", 0), + "chunk_text": doc, + "score": round(1.0 - dist, 4), # cosine distance → similarity + }) + + return chunks + + +def get_embed_status(vault: Path) -> dict: + """Get vector DB status.""" + db_path = get_vector_db_path(vault) + exists = db_path.exists() + chunk_count = 0 + if exists: + try: + collection = get_collection(vault) + chunk_count = collection.count() + except Exception: + pass + + settings = _read_plugin_settings(vault) + return { + "db_exists": exists, + "chunk_count": chunk_count, + "model": settings.get("vector_db_model", "BAAI/bge-small-en-v1.5"), + "mode": settings.get("vector_db_mode", "local"), + }