mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
feat(discussion): remove JSON output and markdown escaping
This commit is contained in:
Research Assistant
parent
a599cd930b
commit
add220d93c
24 changed files with 4774 additions and 352 deletions
359
.planning/intel/ARCH.md
Normal file
359
.planning/intel/ARCH.md
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
# Architecture
|
||||
|
||||
**Analysis Date:** 2026-05-16
|
||||
|
||||
## Architecture Pattern Overview
|
||||
|
||||
**Pattern:** Contract-Driven, Python-writes-JS-reads layered architecture with compound skill agent model.
|
||||
|
||||
**Key Characteristics:**
|
||||
- **Python as data authority** — Python writes canonical JSON snapshots; JS reads them directly (no inference, no SQLite in JS)
|
||||
- **PFResult contract** — All CLI commands return structured `{ok, command, version, data, error}` JSON via `paperforge/core/result.py`
|
||||
- **Atoms -> Molecules -> Compound skill hierarchy** in agent layer
|
||||
- **Three runtime state files** bridge Python backend to JS frontend
|
||||
- **Obsidian plugin** is a thin UI layer that spawns Python subprocesses via `execFile()`/`spawn()`
|
||||
|
||||
## Layers
|
||||
|
||||
### Layer 1: CLI/Plugin Boundary (PFResult Contract)
|
||||
|
||||
**Shared Contract:**
|
||||
- Python commands return `PFResult` dataclass: `{ok, command, version, data, error, warnings, next_actions}`
|
||||
- JSON output via `--json` flag on all commands
|
||||
- Plugin reads either PFResult JSON or falls back to snapshot files
|
||||
- Defined in `paperforge/core/result.py` (69 lines)
|
||||
|
||||
**Error Codes:**
|
||||
- Centralized enum in `paperforge/core/errors.py` (60 lines)
|
||||
- 7 groups: Runtime, Config/Vault, BBT/Zotero, OCR, Sync, Schema, Generic
|
||||
- `.missing_()` handler gracefully handles unknown codes from newer versions
|
||||
|
||||
### Layer 2: Python CLI Backend
|
||||
|
||||
**Entry Point:**
|
||||
- `paperforge/cli.py` (572 lines) — argparse-based CLI with subcommands:
|
||||
- `sync`, `ocr`, `doctor`, `repair`, `status`, `embed`, `memory`, `deep-reading`, `runtime-health`, `dashboard`, `search`, `retrieve`, `paper-context`, `agent-context`, `paper-status`, `reading-log`, `project-log`, `context`
|
||||
|
||||
**CLI Dispatch (`paperforge/commands/`) — 17 command modules:**
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `sync.py` | 62 | Sync CLI wrapper (delegates to SyncService) |
|
||||
| `ocr.py` | 151 | OCR CLI wrapper |
|
||||
| `status.py` | 33 | Status CLI wrapper |
|
||||
| `embed.py` | 242 | Embed build/status/stop CLI |
|
||||
| `memory.py` | 118 | Memory DB build/status CLI |
|
||||
| `dashboard.py` | 191 | Dashboard JSON provider |
|
||||
| `deep.py` | 60 | Deep-reading queue viewer |
|
||||
| `repair.py` | 79 | Repair CLI wrapper |
|
||||
| `runtime_health.py` | 38 | Health check wrapper + snapshot writer |
|
||||
| `reading_log.py` | 470 | Reading log management |
|
||||
| `project_log.py` | 154 | Project log management |
|
||||
| `search.py` | 74 | FTS + vector search |
|
||||
| `retrieve.py` | 66 | Vector retrieval |
|
||||
| `paper_context.py` | 114 | Single-paper context builder |
|
||||
| `agent_context.py` | 85 | Full agent context builder |
|
||||
| `context.py` | 140 | Workspace context |
|
||||
| `paper_status.py` | 63 | Paper status queries |
|
||||
|
||||
**Worker Layer (`paperforge/worker/`) — 16 modules:**
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `ocr.py` | 1835 | PaddleOCR integration (largest module) |
|
||||
| `sync.py` | 1126 | Selection sync + workspace migration |
|
||||
| `status.py` | 1072 | Doctor + status run (health checks) |
|
||||
| `asset_index.py` | 610 | Formal-library.json index builder |
|
||||
| `base_views.py` | 475 | Obsidian Base view generation |
|
||||
| `repair.py` | 389 | Three-way state divergence repair |
|
||||
| `discussion.py` | 331 | Discussion JSON management |
|
||||
| `paper_resolver.py` | 323 | Paper path resolution |
|
||||
| `update.py` | 301 | Self-update mechanism |
|
||||
| `deep_reading.py` | 121 | Deep reading queue |
|
||||
| `_utils.py` | 232 | Shared utilities (JSON I/O, filename slug) |
|
||||
| `asset_state.py` | 177 | Lifecycle/maturity/next-step computation |
|
||||
| `_domain.py` | 78 | Domain config loading |
|
||||
| `_progress.py` | 15 | CLI progress spinner |
|
||||
| `_retry.py` | 69 | Tenacity-based retry helpers |
|
||||
| `vector_db.py` | 70 | Preflight checks for vector DB |
|
||||
| `paper_meta.py` | 70 | Per-paper metadata |
|
||||
|
||||
**Adapter Layer (`paperforge/adapters/`) — 4 modules:**
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `bbt.py` | 210 | BBT JSON export parsing, attachment normalization, PDF identification |
|
||||
| `obsidian_frontmatter.py` | 306 | Frontmatter reading/writing, candidate markdown generation |
|
||||
| `zotero_paths.py` | 54 | Path normalization, wikilink generation |
|
||||
| `collections.py` | 24 | Collection lookup building |
|
||||
|
||||
**Service Layer (`paperforge/services/`) — 2 modules:**
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `sync_service.py` | 264 | Full sync orchestration (load, select, index, clean) |
|
||||
| `skill_deploy.py` | 73 | Agent skill deployment |
|
||||
|
||||
**Memory Layer (`paperforge/memory/`) — 14 modules:**
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `vector_db.py` | 296 | ChromaDB management, embedding, retrieval |
|
||||
| `builder.py` | 238 | SQLite rebuild from canonical index |
|
||||
| `runtime_health.py` | 185 | Layer-by-layer health checks |
|
||||
| `schema.py` | 198 | SQLite schema (DLL), FTS5 config |
|
||||
| `permanent.py` | 164 | JSONL reading-log/project-log/correction-log imports |
|
||||
| `query.py` | 160 | SQL query helpers |
|
||||
| `state_snapshot.py` | 61 | Canonical snapshot writer (3 JSON files) |
|
||||
| `chunker.py` | 73 | OCR fulltext chunking for embedding |
|
||||
| `db.py` | 28 | SQLite connection management (WAL mode) |
|
||||
| `_columns.py` | 38 | Paper table column definitions |
|
||||
| `fts.py` | 84 | FTS5 search |
|
||||
| `events.py` | 86 | Event logging |
|
||||
| `context.py` | 80 | Context assembly |
|
||||
| `refresh.py` | 111 | Refresh logic |
|
||||
|
||||
**Setup Layer (`paperforge/setup/`) — 6 modules:**
|
||||
| File | Lines | Purpose |
|
||||
|------|-------|---------|
|
||||
| `vault.py` | 129 | Vault directory creation |
|
||||
| `runtime.py` | 91 | Python runtime detection |
|
||||
| `agent.py` | 72 | Agent skill file deployment |
|
||||
| `plan.py` | 79 | Installation plan generation |
|
||||
| `config_writer.py` | 68 | Configuration writing |
|
||||
| `checker.py` | 53 | Pre-installation checks |
|
||||
|
||||
**Schema & Doctor:**
|
||||
- `paperforge/schema/field_registry.yaml` — 44 field definitions
|
||||
- `paperforge/schema/__init__.py` — 22 lines: load_field_registry()
|
||||
- `paperforge/doctor/field_validator.py` — 148 lines: frontmatter validation against registry
|
||||
|
||||
**Other Python Modules:**
|
||||
- `paperforge/config.py` (346 lines) — Vault config management, path resolution
|
||||
- `paperforge/setup_wizard.py` (794 lines) — Interactive setup wizard (Obsidian-style)
|
||||
- `paperforge/pdf_resolver.py` (91 lines) — Zotero PDF path resolver
|
||||
- `paperforge/ocr_diagnostics.py` (250 lines) — OCR troubleshooting
|
||||
- `paperforge/logging_config.py` (53 lines) — Logging setup
|
||||
|
||||
### Layer 3: Obsidian Plugin (JS Frontend)
|
||||
|
||||
**Single Monolithic File:** `paperforge/plugin/main.js` (4,914 lines)
|
||||
|
||||
**Key Components:**
|
||||
1. **`memoryState` IIFE (lines 8-143)** — Inline module for reading snapshots, python detection, buildSnapshot()
|
||||
2. **Inlined testable functions (lines 146-341)** — `resolvePythonExecutable()`, `checkRuntimeVersion()`, `classifyError()`, `buildRuntimeInstallCommand()`, `parseRuntimeStatus()`, `runSubprocess()`, `buildCommandArgs()`, ACTIONS array
|
||||
3. **Cross-platform Python/BBT detection (lines 343-509)** — `paperforgeEnrichedEnv()`, `getPaperforgePythonCmd()`, `tryExecPythonVersion()`, Mac/Linux/Windows Python discovery, BetterBibTeX profile scanning
|
||||
4. **i18n System (lines 524-817)** — `LANG` object with `en` and `zh` keys, `t()` translation function, `langFromApp()` auto-detection
|
||||
5. **PaperForgeStatusView (lines 883-2515)** — Obsidian ItemView with 3 modes:
|
||||
- `global` mode — system homepage with library snapshot, system status grid, issues panel, contextual actions
|
||||
- `paper` mode — per-paper reading companion with workflow toggles, next-step card, discussion, technical details
|
||||
- `collection` mode — domain-level workflow overview with funnel, OCR pipeline, batch actions
|
||||
6. **PaperForgeSettingTab (lines 2517-3903)** — Settings UI with two tabs:
|
||||
- `setup` tab — Python interpreter management, runtime health, install wizard, preparation guide
|
||||
- `features` tab — Skills management (system/user), Memory Layer, Vector DB config with full lifecycle
|
||||
7. **Setup Wizard Modal** — 5-step interactive installation wizard
|
||||
8. **PaperForgePlugin class (lines ~4470-4912)** — Plugin registration, commands, auto-update, file polling
|
||||
9. **OCR Privacy Modal** (lines 3905-3939) — Once-per-session privacy warning
|
||||
|
||||
**Extracted Pure Functions:**
|
||||
- `paperforge/plugin/src/testable.js` (224 lines) — 8 exported functions from main.js, no Obsidian dependency
|
||||
|
||||
### Layer 4: Agent Skill Layer
|
||||
|
||||
**Compound skill model:**
|
||||
- `paperforge/skills/paperforge/SKILL.md` — Compound: bootstrap + agent-context + runtime-health -> route -> molecule
|
||||
- `paperforge/skills/paperforge/workflows/` — 8 molecule workflow files: paper-search, deep-reading, paper-qa, reading-log, project-log, methodology, project-engineering
|
||||
- `paperforge/skills/paperforge/references/` — Shared reference materials including 19 chart-reading guides
|
||||
- `paperforge/skills/paperforge/scripts/` — Atomic scripts: `pf_bootstrap.py` (227 lines), `pf_deep.py` (1,569 lines)
|
||||
|
||||
**Supported Agent Platforms:**
|
||||
- OpenCode (primary), Claude Code, Cursor, Windsurf, GitHub Copilot, Codex, Cline
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Data Flow 1: Snapshot Bridge (Python -> JS)
|
||||
|
||||
```
|
||||
Python writes (at end of every relevant CLI command):
|
||||
paperforge/commands/runtime_health.py -> write_runtime_health()
|
||||
-> System/PaperForge/indexes/runtime-health.json
|
||||
|
||||
paperforge/worker/status.py (run_status) -> write_memory_runtime()
|
||||
-> System/PaperForge/indexes/memory-runtime-state.json
|
||||
|
||||
paperforge/commands/embed.py (embed status) -> write_vector_runtime()
|
||||
-> System/PaperForge/indexes/vector-runtime-state.json
|
||||
|
||||
JS reads (via memoryState IIFE):
|
||||
main.js:30-35 -> readJSONFile() -> memoryState.getMemoryRuntime/VectorRuntime/RuntimeHealth()
|
||||
|
||||
Snapshot Contract (state_snapshot.py):
|
||||
memory-runtime-state.json: {schema_version, updated_at, source_command, paper_count_db, paper_count_index, fresh, needs_rebuild, last_full_build_at, schema_version_db, fts_ready}
|
||||
|
||||
vector-runtime-state.json: {schema_version, updated_at, source_command, enabled, mode, model, deps_installed, deps_missing, py_version, db_exists, chunk_count, build_state}
|
||||
|
||||
runtime-health.json: {summary: {status, reason, safe_read, safe_write, safe_build, safe_vector}, layers: {bootstrap, read, write, index, vector}, capabilities}
|
||||
```
|
||||
|
||||
### Data Flow 2: Plugin Action Execution
|
||||
|
||||
```
|
||||
User clicks button in Dashboard (e.g., "Sync Library")
|
||||
-> PaperForgeStatusView._runAction(action, button) [line 2274]
|
||||
-> guard checks: OCR privacy (DASH-03), disabled, double-click prevention
|
||||
-> resolvePythonExecutable(vaultPath, settings)
|
||||
-> spawn(pythonExe, [...extraArgs, '-m', 'paperforge', action.cmd, ...extraArgs])
|
||||
-> stdout event listener parses log lines, updates message area
|
||||
-> 4-second polling timer: _fetchStats(true)
|
||||
-> on close: check exit code, show success/failure, refresh stats
|
||||
-> triggers: _fetchStats() -> tries 'paperforge dashboard --json' first, falls back to formal-library.json, falls back to 'paperforge status --json'
|
||||
```
|
||||
|
||||
### Data Flow 3: Memory Build
|
||||
|
||||
```
|
||||
CLI: paperforge memory build
|
||||
-> paperforge/commands/memory.py -> build_from_index(vault)
|
||||
-> paperforge/worker/asset_index.py -> read_index(vault) # reads formal-library.json
|
||||
-> paperforge/memory/builder.py -> build_from_index(vault):
|
||||
1. Compute canonical hash from sorted index items
|
||||
2. Open SQLite connection to paperforge.db
|
||||
3. Schema version check: drop + recreate if mismatched
|
||||
4. DELETE all tables (foreign_keys OFF)
|
||||
5. Insert papers, assets, aliases rows
|
||||
6. FTS5 population: INSERT INTO paper_fts FROM papers
|
||||
7. Import JSONL logs: reading-log.jsonl -> reading_log table
|
||||
8. Import JSONL logs: project-log.jsonl -> project_log table
|
||||
9. Import corrections: correction-log.jsonl -> paper_events table
|
||||
10. Write meta table: schema_version, paperforge_version, created_at, last_full_build_at, canonical_index_hash
|
||||
11. COMMIT
|
||||
```
|
||||
|
||||
### Data Flow 4: Embed Build
|
||||
|
||||
```
|
||||
CLI: paperforge embed build
|
||||
-> paperforge/commands/embed.py run():
|
||||
1. Preflight check via _preflight_check() (deps installed, mode valid)
|
||||
2. Read canonical index from asset_index
|
||||
3. Filter items where ocr_status == "done"
|
||||
4. Mark build state as "running" in vector-build-state.json (pid, model, mode, total)
|
||||
5. For each paper:
|
||||
a. Read OCR fulltext from fulltext_path
|
||||
b. chunk_fulltext() -> split by page, group paragraphs, detect sections
|
||||
c. delete_paper_vectors(vault, key) -> remove old chunks from ChromaDB
|
||||
d. embed_paper() -> either local (sentence-transformers) or API (openai)
|
||||
e. Mark progress: EMBED_PROGRESS:current:total:key
|
||||
6. On completion: mark build state "completed", write vector-runtime-state.json
|
||||
```
|
||||
|
||||
### Data Flow 5: Sync (Full Lifecycle)
|
||||
|
||||
```
|
||||
CLI: paperforge sync
|
||||
-> paperforge/commands/sync.py -> SyncService.run():
|
||||
Phase 1: Selection
|
||||
-> load_exports() -> load BBT JSON files from System/PaperForge/exports/
|
||||
-> run_selection_sync() -> legacy worker sync
|
||||
Phase 2: Index
|
||||
-> resolve_paths() + load_domain_config()
|
||||
-> ensure_base_views() -> generate .base files
|
||||
-> migrate_to_workspace() -> flat notes -> workspace dirs
|
||||
-> asset_index.build_index() -> generate formal-library.json
|
||||
-> clean_orphaned_records() + clean_flat_notes()
|
||||
Phase 3: Return PFResult
|
||||
-> {... selection_result, index_result }
|
||||
```
|
||||
|
||||
### Data Flow 6: Snapshot Refresh Polling
|
||||
|
||||
```
|
||||
Plugin startup (_startFilePolling, line 4681):
|
||||
-> setInterval every 120 seconds:
|
||||
-> _checkExports(): monitor mtime of export JSON files, trigger auto-sync on change
|
||||
-> _checkOcr(): monitor mtime of meta.json files, trigger per-paper sync on change
|
||||
|
||||
First launch detection (line 4627):
|
||||
-> Check if memory-runtime-state.json exists
|
||||
-> If missing: fire-and-forget 'runtime-health --json' to generate snapshots
|
||||
|
||||
Formal-library.json change detection (line 2493):
|
||||
-> active-leaf-change event: 300ms debounce, detect mode (global/paper/collection)
|
||||
-> vault.modify event: if file is formal-library.json -> invalidate cache + refresh mode
|
||||
```
|
||||
|
||||
## State Management
|
||||
|
||||
**Three-way state model** (managed by repair worker):
|
||||
1. **Canonical index** (`formal-library.json`) — the "source of truth" for all paper metadata
|
||||
2. **Formal notes** (frontmatter per .md file) — user-controlled workflow flags (do_ocr, analyze)
|
||||
3. **SQLite DB** (`paperforge.db`) — denormalized, FTS-enabled query layer
|
||||
|
||||
**Divergence detection:**
|
||||
- `paperforge/worker/repair.py` (389 lines) — three-way state divergence repair
|
||||
- `paperforge/worker/asset_state.py` (177 lines) — lifecycle/maturity/next-step computation
|
||||
- `paperforge/worker/asset_index.py` (610 lines) — canonical index builder
|
||||
|
||||
## Filesystem State Files
|
||||
|
||||
```
|
||||
System/PaperForge/indexes/
|
||||
formal-library.json — Canonical index (generated by sync)
|
||||
memory-runtime-state.json — Memory layer snapshot (generated by status, memory, embed)
|
||||
vector-runtime-state.json — Vector DB snapshot (generated by embed status/build)
|
||||
runtime-health.json — Runtime health summary (generated by runtime-health)
|
||||
vector-build-state.json — Embed build progress (streamed by embed build)
|
||||
paperforge.db — SQLite memory layer database
|
||||
vectors/ — ChromaDB persistence directory
|
||||
```
|
||||
|
||||
## Cross-Platform Handling
|
||||
|
||||
- Python interpreter detection: Windows `Scripts/` vs POSIX `bin/` in venv detection
|
||||
- Zotero data dir detection: APPDATA (Win) vs Application Support (macOS) vs .zotero (Linux)
|
||||
- Junction vs symlink: Windows junctions handled via ctypes in `worker/status.py`
|
||||
- Git detection: `cmd.exe /c where git` (Win) vs `which git` (POSIX)
|
||||
- UTF-8 console: Win32 SetConsoleOutputCP(65001)
|
||||
- Enriched PATH: homebrew (macOS), .local/bin (Linux), git dir appended
|
||||
|
||||
## Error Handling Strategy
|
||||
|
||||
**Python Side:**
|
||||
- PFResult contract -> `data.error` field with ErrorCode
|
||||
- Recovery hints: `data.next_actions: [dict]`, `error.suggestions: [str]`
|
||||
- Per-module error classification via ErrorCode enum (7 groups)
|
||||
- repair command handles three-way state divergence
|
||||
|
||||
**JS Side:**
|
||||
- `classifyError()` (line 222) — maps error codes to {type, message, recoverable, action}
|
||||
- `parseRuntimeStatus()` (line 248) — parses stderr/stdout for known error patterns
|
||||
- 3-tier fallback for stats: `paperforge dashboard --json` -> formal-library.json -> `paperforge status --json`
|
||||
- OCR privacy notice (once per session)
|
||||
- Double-click guard on action buttons
|
||||
- Disabled action resolution
|
||||
|
||||
## i18n System
|
||||
|
||||
**Location:** `paperforge/plugin/main.js`, lines 524-817
|
||||
|
||||
**Mechanism:**
|
||||
- `LANG` object contains `en` and `zh` keys
|
||||
- `Object.assign(LANG.en, ...)` extends both EN and ZH with shared fields
|
||||
- `langFromApp(app)` checks Obsidian config -> localStorage -> defaults to 'en'
|
||||
- `t(key)` function: returns `T[key]` || `LANG.en[key]` || key as fallback
|
||||
- Translation granularity: per-UI-element level (~400 translated strings)
|
||||
- Features tab has full bilingual support (field labels, descriptions, button text, wizard steps)
|
||||
|
||||
## Plugin Registration
|
||||
|
||||
**Location:** `paperforge/plugin/main.js`, lines ~4470-4912
|
||||
|
||||
- Plugin ID: `paperforge`
|
||||
- Custom view: `VIEW_TYPE_PAPERFORGE = 'paperforge-status'`
|
||||
- Custom icon: `PF_ICON_ID = 'paperforge'` with SVG path data
|
||||
- 5 Obsidian commands registered:
|
||||
- `paperforge-open-view` — Open Dashboard
|
||||
- `paperforge-sync` — Sync Library
|
||||
- `paperforge-ocr` — Run OCR
|
||||
- `paperforge-doctor` — Run Doctor
|
||||
- `paperforge-repair` — Repair Issues
|
||||
- Auto-update: checks plugin version vs installed package, tries PyPI then git
|
||||
|
||||
---
|
||||
|
||||
*Architecture analysis: 2026-05-16*
|
||||
201
.planning/intel/QUALITY.md
Normal file
201
.planning/intel/QUALITY.md
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
# Quality Assessment
|
||||
|
||||
**Analysis Date:** 2026-05-16
|
||||
|
||||
## Code Organization Strengths
|
||||
|
||||
**Well-Defined Module Boundaries:**
|
||||
- Clear 5-layer architecture (commands/ -> services/ -> adapters/ -> core/ -> worker/) enforced by v2.1 refactor
|
||||
- Adapter layer (`paperforge/adapters/`) is genuinely independent and testable — BBT parsing, frontmatter, path normalization are isolated from business logic
|
||||
- Core contract types (`PFResult`, `ErrorCode`) are small, focused, and widely used
|
||||
- Worker modules (`paperforge/worker/`) separate "mechanical labor" from orchestration
|
||||
|
||||
**Version Management:**
|
||||
- Single source of truth in `paperforge/__init__.py`
|
||||
- `scripts/bump.py` automates across 4 files + git tag
|
||||
- Consistent `__version__` import pattern throughout codebase
|
||||
|
||||
**PFResult Contract Compliance:**
|
||||
- All 17 command modules return PFResult
|
||||
- JSON output flag (`--json`) is consistent across all commands
|
||||
- Plugin has uniform fallback chain: PFResult JSON -> snapshot files -> CLI fallback
|
||||
- ErrorCode enum has `.missing_()` handler for forward compatibility
|
||||
|
||||
**Test Architecture (Python):**
|
||||
- 7 test directories with clear levels: unit, cli, integration, e2e, journey, chaos, audit
|
||||
- ~93 Python test files, ~13,279 total lines
|
||||
- Snapshot regression tests for JSON contracts
|
||||
- Audit tests validate mock fidelity against real pipeline output
|
||||
- Ruff linting enforced via pre-commit (E, F, I, UP, B, SIM)
|
||||
- Per-file ruff ignore rules are documented with reasons
|
||||
|
||||
## Code Organization Risks
|
||||
|
||||
### CRITICAL: main.js Monolith (4,914 lines)
|
||||
|
||||
**File:** `paperforge/plugin/main.js`
|
||||
|
||||
**Issues:**
|
||||
- Single file contains: IIFE module, 10+ pure functions, ItemView class (~1600 lines), SettingTab class (~1400 lines), Modal classes, and the Plugin class
|
||||
- No JS build system — raw JS shipped directly
|
||||
- Inline duplication: `memoryState` IIFE (lines 8-143) duplicates `resolvePythonExecutable()` logic from `src/testable.js` (lines 148-188) — two implementations of the same detection algorithm
|
||||
- `memoryState.getCachedPython()` (line 85) uses `var` scoping and caches result differently from the pure `resolvePythonExecutable()` function
|
||||
- Hard to test: most UI logic cannot be unit-tested because it depends on Obsidian `app` object
|
||||
- No module system: `memoryState`, inline functions, and class definitions all share global scope
|
||||
|
||||
**Mitigation:**
|
||||
- Extraction to `src/testable.js` has started (224 lines, 8 exported functions)
|
||||
- 4 vitest test files exist for testable.js (310 lines)
|
||||
- Plan 53-001 documented extraction strategy
|
||||
|
||||
### MODERATE: Duplicate Python Detection Logic
|
||||
|
||||
**Impact:** Python executable resolution is implemented in at least 4 locations:
|
||||
1. `memoryState.getCachedPython()` in `main.js` lines 84-115 (vault-scoped caching)
|
||||
2. `resolvePythonExecutable()` in `main.js` lines 148-188 (testable export, no caching)
|
||||
3. `getPaperforgePythonCmd()` in `main.js` lines 405-434 (macOS-specific)
|
||||
4. `_resolve_plugin_interpreter()` in `worker/status.py` lines 235-289 (Python reimplementation)
|
||||
|
||||
**Risk:** These diverged implementations may behave differently for edge cases (venv detection, macOS stub python, Windows py launcher).
|
||||
|
||||
### MODERATE: Snapshot File Race Conditions
|
||||
|
||||
**Files:** `memory-runtime-state.json`, `vector-runtime-state.json`, `runtime-health.json`
|
||||
|
||||
**Analysis:**
|
||||
- Multiple Python commands write to the same snapshot files (status, embed, memory, runtime-health)
|
||||
- Plugin reads these files synchronously with `readJSONFile()` (uses `fs.existsSync` + `fs.readFileSync`)
|
||||
- No file locking in Python writers — two concurrent CLI invocations could:
|
||||
- Interleave writes to the same file
|
||||
- Write partial JSON (plugin reads incomplete file -> parse error -> returns null)
|
||||
- Worker/vector_db.py has atomic write for build state (write to .tmp then rename) but state_snapshot.py does NOT use this pattern
|
||||
|
||||
**Recommendation:** Extend atomic write pattern (write.tmp + replace) to all three snapshot files. Or use a simple file lock (paperforge uses `filelock` in pyproject.toml already for other purposes).
|
||||
|
||||
**Evidence:**
|
||||
- `state_snapshot.py` writes directly: `path.write_text(json.dumps(snap))` — no atomic write
|
||||
- `vector_db.py` uses atomic write: `tmp = path.with_suffix(".tmp"); tmp.write_text(); tmp.replace(path)` — correct pattern
|
||||
|
||||
### LOW: Subprocess Error Handling Gaps
|
||||
|
||||
**File:** `paperforge/plugin/main.js` line 2274 `_runAction()`
|
||||
|
||||
**Analysis:**
|
||||
- stdout/stderr parsed line-by-line with rudimentary filtering
|
||||
- No structured parsing of PFResult output in action runner (contrast with SettingsTab._callPython which parses exit codes)
|
||||
- `spawn` timeout = 600000ms (10 min) for sync — no user-facing progress for the first 4 seconds (pollTimer)
|
||||
- `_autoSync` uses `exec()` (shell-unsafe cmd string building) rather than `execFile()` (safer)
|
||||
- Line 4720: `` const cmd = `"${pyResult.path}" -m paperforge --vault "${vaultPath}" sync`; ``
|
||||
- If vaultPath contains spaces or special characters, this is a shell injection vector (low severity since vault path is user-controlled but not attacker-controlled)
|
||||
|
||||
### MODERATE: I18n Coverage Gaps
|
||||
|
||||
**File:** `paperforge/plugin/main.js` lines 524-817
|
||||
|
||||
**Analysis:**
|
||||
- Dashboard UI (PaperForgeStatusView) uses many hardcoded English strings:
|
||||
- Line 1559: `"System Status"`, line 1572: `"Library Snapshot"`, lines 1575-1578: `"papers"`, `"PDFs ready"`, `"OCR done"`, `"deep-read done"`
|
||||
- Lines 2163-2182: `"Workflow Overview"`, `"Total"`, `"PDF Ready"`, `"OCR Done"`, `"Deep Read"`, `"Sync Library"`, `"Run OCR"`
|
||||
- Line 2219: `"Attention"`, line 3577: `" chunks embedded"`
|
||||
- Settings tab uses `t()` function but StatusView UI is mostly untranslated
|
||||
- Translation structure is flat (all keys in one object) — hard to maintain
|
||||
- ~400 keys total but StatusView covers <50 via `t()`
|
||||
|
||||
### LOW: Settings Tab Complexity
|
||||
|
||||
**File:** `paperforge/plugin/main.js` lines 2517-3903 (PaperForgeSettingTab)
|
||||
|
||||
**Analysis:**
|
||||
- 1,386 lines for a single settings tab class
|
||||
- Mixes: state management, UI rendering, Python subprocess calls, file I/O, skill management, vector DB config, model cache management
|
||||
- Inline CSS strings throughout (`element.style.cssText = ...`) — hard to maintain
|
||||
- State tracking intertwined with rendering (`_vectorDepsOk`, `_embedStatusText`, `_memoryStatusText` are both state flags and cache markers)
|
||||
|
||||
### MODERATE: JS Test Coverage
|
||||
|
||||
- 4 test files for JS, 310 lines total
|
||||
- Only test `src/testable.js` exported functions
|
||||
- Zero tests for: PaperForgeStatusView, PaperForgeSettingTab, Plugin class, memoryState IIFE, i18n system
|
||||
- No E2E tests for plugin behavior
|
||||
- Test setup uses vitest but no Obsidian API mocking utilities
|
||||
|
||||
### HIGH: Python Test Coverage Distribution
|
||||
|
||||
**Total:** ~13,279 lines of tests across 93 files
|
||||
|
||||
**Distribution concerns:**
|
||||
- Tests live in 7 levels (unit, cli, integration, e2e, journey, chaos, audit) but no test count per level collected here
|
||||
- Largest worker modules (ocr.py: 1835 lines, sync.py: 1126 lines, status.py: 1072 lines) are the hardest to unit test
|
||||
- OCR testing requires PaddleOCR API mock (responses library used, but OCR module is 1835 lines — likely under-tested)
|
||||
- No test coverage target documented (not enforced in CI)
|
||||
|
||||
## Technical Debt Inventory
|
||||
|
||||
### Debt 1: Legacy Worker Patterns
|
||||
**Files:** `paperforge/worker/sync.py` (1126 lines), `paperforge/worker/ocr.py` (1835 lines)
|
||||
|
||||
**Issues:**
|
||||
- v2.1 architecture introduced SyncService but worker/sync.py still contains 1126 lines of legacy logic
|
||||
- SyncService at `services/sync_service.py` line 37: "Full migration of note-writing logic is planned for v2.2"
|
||||
- `_retry.py` module exists but not consistently used
|
||||
|
||||
### Debt 2: setup_wizard.py Monolith
|
||||
**File:** `paperforge/setup_wizard.py` (794 lines)
|
||||
|
||||
**Issues:**
|
||||
- Setup wizard split into `paperforge/setup/` (6 modules, ~500 lines total) but the legacy `setup_wizard.py` still exists at 794 lines
|
||||
- Likely duplicated logic between setup/ modules and monolithic setup_wizard.py
|
||||
|
||||
### Debt 3: Ruff Per-File Ignores
|
||||
**File:** `pyproject.toml` lines 99-113
|
||||
|
||||
**Issues:**
|
||||
- 7 files have per-file ruff rule ignores
|
||||
- `F821` (undefined names) ignored in update.py and sync.py — actual bugs or intentional?
|
||||
- 8 test-level rules ignored across all test files (E501, E402, etc.)
|
||||
|
||||
### Debt 4: No CI/CD Integration
|
||||
- No GitHub Actions workflow detected
|
||||
- Tests run locally only
|
||||
- Release process is manual (`gh release create`)
|
||||
- No automated test run on PR
|
||||
|
||||
### Debt 5: Obsidian API Version Constraint
|
||||
**File:** `paperforge/plugin/manifest.json`
|
||||
|
||||
- `minAppVersion: "1.9.0"` — relatively recent, may not be compatible with older Obsidian installs
|
||||
- Desktop-only (`"isDesktopOnly": true`) — Electron APIs assumed
|
||||
|
||||
## Dependency Risks
|
||||
|
||||
### Risk 1: PaddleOCR API Dependency
|
||||
- OCR pipeline requires external PaddleOCR API (Baidu)
|
||||
- API key required (`PADDLEOCR_API_TOKEN` in .env)
|
||||
- No offline OCR fallback
|
||||
- Chinese service may have latency/availability issues outside China
|
||||
|
||||
### Risk 2: ChromaDB + sentence-transformers Optional Dependencies
|
||||
- Vector DB feature requires 3 optional packages (~500MB)
|
||||
- sentence-transformers downloads models on first use (80-440MB each)
|
||||
- HF mirror dependency for users behind firewalls
|
||||
- No fallback to pure-JS embedding
|
||||
|
||||
### Risk 3: Electron require() Limitations
|
||||
- Obsidian plugins run in Electron renderer process
|
||||
- `require('node:child_process')`, `require('fs')` work but are Electron-specific
|
||||
- No `require()` for local project files — forces all code into main.js or inline IIFE
|
||||
- This is the root cause of the main.js monolith
|
||||
|
||||
## Recommendations
|
||||
|
||||
1. **Split main.js:** Extract SettingTab, StatusView, and Plugin class into separate files, use a manual concatenation build step
|
||||
2. **Atomic snapshot writes:** Extend .tmp + rename pattern to all 3 snapshot files
|
||||
3. **Deduplicate Python detection:** Single shared algorithm with documented fallback order
|
||||
4. **Translate StatusView:** Wrap all user-facing strings in plugin StatusView with `t()` calls
|
||||
5. **Add JS UI tests:** Mock Obsidian API for StatusView rendering tests
|
||||
6. **Remove legacy setup_wizard.py:** After confirming setup/6-module split is complete
|
||||
7. **Add CI pipeline:** GitHub Actions for ruff + pytest + vitest
|
||||
|
||||
---
|
||||
|
||||
*Quality assessment: 2026-05-16*
|
||||
93
.planning/intel/TECH.md
Normal file
93
.planning/intel/TECH.md
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
# Technology Stack
|
||||
|
||||
**Analysis Date:** 2026-05-16
|
||||
|
||||
## Languages
|
||||
|
||||
**Primary:**
|
||||
- **Python 3.10+** — CLI backend for all data-heavy operations: sync, OCR, memory build, embed build, repair, status, runtime health, agent context. 71 Python modules totaling ~16,700 lines (excluding node_modules, test fixture files).
|
||||
- Entry point: `paperforge/__main__.py` -> `paperforge/cli.py` (572 lines)
|
||||
- Version defined in single source: `paperforge/__init__.py` (__version__ = "1.5.6rc3")
|
||||
|
||||
**Secondary:**
|
||||
- **JavaScript (Node.js / Obsidian API)** — Obsidian plugin for UI, state display, subprocess orchestration. 2 source files + 4 test files.
|
||||
- `paperforge/plugin/main.js` — 4,914 lines (monolithic, single file)
|
||||
- `paperforge/plugin/src/testable.js` — 224 lines (extracted pure functions)
|
||||
- `paperforge/plugin/tests/*.test.mjs` — 4 test files, 310 lines total
|
||||
|
||||
## Runtimes
|
||||
|
||||
**Python Runtime:**
|
||||
- CPython 3.10+
|
||||
- Package manager: pip
|
||||
- Package: `paperforge` (installed via pip or git+https://github.com/LLLin000/PaperForge.git)
|
||||
- Lockfile: Not detected (pip-only, no requirements.lock)
|
||||
|
||||
**JS Runtime:**
|
||||
- Node.js (embedded in Obsidian/Electron)
|
||||
- Package manager: npm (dev-only, vitest)
|
||||
- Package config: Not standalone (Obsidian plugin manifest at `paperforge/plugin/manifest.json`)
|
||||
- Obsidian API version: minAppVersion 1.9.0
|
||||
|
||||
## Frameworks
|
||||
|
||||
**Core Python Libraries:**
|
||||
| Library | Version | Purpose |
|
||||
|---------|---------|---------|
|
||||
| requests | >=2.31.0 | HTTP calls (PaddleOCR API) |
|
||||
| pymupdf | >=1.23.0 | PDF parsing |
|
||||
| Pillow | >=10.0.0 | Image processing |
|
||||
| tenacity | >=8.2.0 | Retry logic for OCR and subprocess calls |
|
||||
| tqdm | >=4.66.0 | Progress bars in CLI |
|
||||
| filelock | >=3.13.0 | Cross-process file locking |
|
||||
| PyYAML | >=6.0 | YAML field registry parsing |
|
||||
| chromadb | >=0.5.0 (optional) | Vector database persistence |
|
||||
| sentence-transformers | >=3.0.0 (optional) | Local embedding models |
|
||||
| openai | >=1.0.0 (optional) | API-mode embeddings |
|
||||
|
||||
**JS Dependencies (Obsidian Plugin):**
|
||||
- `obsidian` (built-in Electron API) — Plugin, Notice, ItemView, Modal, Setting, PluginSettingTab, addIcon
|
||||
- `node:child_process` — exec, execFile, spawn, execFileSync
|
||||
- `fs`, `path`, `os` (Node.js built-in)
|
||||
- **No JS package.json detected for plugin build process** — plugin ships as raw JS
|
||||
|
||||
**Testing:**
|
||||
| Tool | Purpose |
|
||||
|------|---------|
|
||||
| pytest (Python) | Test runner — 173 tests across 6 test levels |
|
||||
| pytest-snapshot | Snapshot regression testing for JSON contracts |
|
||||
| pytest-timeout | Timeout handling for tests |
|
||||
| responses | HTTP mock library |
|
||||
| pytest-mock | Mock support |
|
||||
| coverage | Code coverage measurement |
|
||||
| ruff | Linting + formatting (py310 target, 120 char width) |
|
||||
| vitest (JS) | JS test runner — 4 test files for testable.js |
|
||||
|
||||
**Agent System:**
|
||||
- OpenCode / Claude Code / Cursor / Windsurf / GitHub Copilot — multiple agent platforms supported
|
||||
- Compound skill model: SKILL.md (compound) -> workflows/ (molecules) -> scripts/ (atoms)
|
||||
- Skill files deployed by `paperforge/services/skill_deploy.py`
|
||||
|
||||
## Configuration
|
||||
|
||||
**Environment:**
|
||||
- `.env` at vault root — contains PADDLEOCR_API_TOKEN, OPENAI_API_KEY (never read contents, only existence checked)
|
||||
- Plugin settings in `.obsidian/plugins/paperforge/data.json`
|
||||
- Central config at vault root `paperforge.json` — vault_config block with path overrides, schema_version
|
||||
- Python `paperforge/config.py` (346 lines) — DEFAULT_CONFIG, paperforge_paths(), load_vault_config(), read_paperforge_json(), CONFIG_PATH_KEYS
|
||||
|
||||
**Build Config Files:**
|
||||
- `pyproject.toml` — setuptools build, ruff config, pytest config, optional-dependencies
|
||||
- `ruff.toml` (none — embedded in pyproject.toml)
|
||||
- `.pre-commit-config.yaml` — ruff check + format (in git repo root)
|
||||
- `manifest.json` (plugin) — id, name, version, minAppVersion
|
||||
|
||||
## Version Management
|
||||
|
||||
- Single source of truth: `paperforge/__init__.py` (__version__)
|
||||
- `scripts/bump.py` automates version bump across: `__init__.py`, `paperforge/plugin/manifest.json`, root `manifest.json`, `paperforge/plugin/versions.json`
|
||||
- Release process: bump -> git push --tags -> gh release create with 4 plugin files
|
||||
|
||||
---
|
||||
|
||||
*Stack analysis: 2026-05-16*
|
||||
|
|
@ -0,0 +1,644 @@
|
|||
# PaperForge Stabilization Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Harden PaperForge's runtime contracts, align CLI/skill/doc truth sources, reset documentation IA, and reduce user-facing complexity without a frontend rewrite.
|
||||
|
||||
**Architecture:** Execute four sequential packages. Package A seals runtime and path truth. Package B aligns command and skill truth to the live CLI. Package C rebuilds documentation boundaries around single-audience ownership. Package D reduces first-run complexity by progressive disclosure while preserving the existing frontend architecture.
|
||||
|
||||
**Tech Stack:** Python 3.10+, Obsidian plugin JS, argparse CLI, SQLite, JSON snapshots, Vitest, pytest, GitHub Actions
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
### Runtime Contract Files
|
||||
|
||||
- `paperforge/memory/state_snapshot.py` — canonical runtime snapshot writers
|
||||
- `paperforge/memory/vector_db.py` — existing atomic-write reference for build state
|
||||
- `paperforge/core/io.py` — candidate shared JSON atomic write helper
|
||||
- `paperforge/worker/asset_index.py` — canonical index building and incremental refresh
|
||||
- `paperforge/worker/repair.py` — direct index mutations to be routed through shared safe path
|
||||
- `paperforge/services/sync_service.py` — sync orchestration and final-state semantics
|
||||
- `paperforge/memory/refresh.py` — DB refresh behavior after index changes
|
||||
- `paperforge/commands/embed.py` — build/status/stop lifecycle behavior
|
||||
- `paperforge/commands/runtime_health.py` — startup/runtime snapshot behavior
|
||||
- `paperforge/plugin/main.js` — runtime path/state reads and startup/update behavior
|
||||
|
||||
### Command and Skill Truth Files
|
||||
|
||||
- `paperforge/cli.py` — live command truth source
|
||||
- `paperforge/commands/paper_context.py`
|
||||
- `paperforge/commands/paper_status.py`
|
||||
- `paperforge/commands/search.py`
|
||||
- `paperforge/commands/reading_log.py`
|
||||
- `paperforge/commands/project_log.py`
|
||||
- `paperforge/commands/agent_context.py`
|
||||
- `paperforge/memory/runtime_health.py`
|
||||
- `paperforge/skills/paperforge/SKILL.md`
|
||||
- `paperforge/skills/paperforge/workflows/*.md`
|
||||
|
||||
### Documentation IA Files
|
||||
|
||||
- `README.md`
|
||||
- `README.en.md`
|
||||
- `AGENTS.md`
|
||||
- `docs/COMMANDS.md`
|
||||
- `docs/ARCHITECTURE.md`
|
||||
- `docs/getting-started.md` (new)
|
||||
- `docs/troubleshooting.md` (new)
|
||||
- `docs/maintainer-guide.md` (new)
|
||||
|
||||
### UX Simplification Files
|
||||
|
||||
- `paperforge/plugin/main.js`
|
||||
- `README.md`
|
||||
- `README.en.md`
|
||||
- `docs/getting-started.md`
|
||||
- `docs/troubleshooting.md`
|
||||
|
||||
### Likely Test Files
|
||||
|
||||
- `tests/unit/commands/test_embed.py`
|
||||
- `tests/unit/commands/test_runtime_health.py`
|
||||
- `tests/unit/memory/test_runtime_health.py`
|
||||
- `tests/unit/memory/test_refresh.py`
|
||||
- `tests/test_asset_index.py`
|
||||
- `tests/test_asset_index_integration.py`
|
||||
- `tests/test_repair.py`
|
||||
- `tests/cli/test_json_contracts.py`
|
||||
- `tests/test_command_docs.py`
|
||||
- `tests/journey/test_onboarding.py`
|
||||
- `paperforge/plugin/tests/runtime.test.mjs`
|
||||
- `paperforge/plugin/tests/commands.test.mjs`
|
||||
- `paperforge/plugin/tests/errors.test.mjs`
|
||||
|
||||
---
|
||||
|
||||
## Package A: Runtime Contract Hardening
|
||||
|
||||
### Task A1: Establish Safe Snapshot Write Primitive
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/state_snapshot.py`
|
||||
- Modify: `paperforge/core/io.py`
|
||||
- Test: `tests/unit/commands/test_runtime_health.py`
|
||||
- Test: `tests/unit/commands/test_embed.py`
|
||||
|
||||
- [ ] **Step 1: Add failing tests for contract snapshot writes**
|
||||
|
||||
Add tests that assert snapshot writers use atomic replace semantics rather than direct in-place writes. Prefer behavioral checks around helper calls or final file integrity after interrupted write simulation.
|
||||
|
||||
- [ ] **Step 2: Run targeted tests to confirm current failure or gap**
|
||||
|
||||
Run: `python -m pytest tests/unit/commands/test_runtime_health.py tests/unit/commands/test_embed.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Introduce shared atomic JSON write helper**
|
||||
|
||||
Implement a minimal helper in `paperforge/core/io.py` or a similarly narrow shared location that writes JSON via temp file then replace, with UTF-8 encoding and parent creation only if already consistent with project patterns.
|
||||
|
||||
- [ ] **Step 4: Route runtime snapshot writers through the helper**
|
||||
|
||||
Update `write_memory_runtime()`, `write_vector_runtime()`, and `write_runtime_health()` to use the shared helper.
|
||||
|
||||
- [ ] **Step 5: Re-run targeted tests**
|
||||
|
||||
Run: `python -m pytest tests/unit/commands/test_runtime_health.py tests/unit/commands/test_embed.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 6: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: snapshot writers are atomic and targeted tests pass.
|
||||
|
||||
### Task A2: Unify Canonical Index Mutation Path
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/core/io.py`
|
||||
- Modify: `paperforge/worker/asset_index.py`
|
||||
- Modify: `paperforge/worker/repair.py`
|
||||
- Test: `tests/test_asset_index.py`
|
||||
- Test: `tests/test_asset_index_integration.py`
|
||||
- Test: `tests/test_repair.py`
|
||||
|
||||
- [ ] **Step 1: Add failing tests for overlapping index mutation scenarios**
|
||||
|
||||
Cover at least:
|
||||
- refresh-style update preserves unrelated entries
|
||||
- repair-style update does not bypass shared mutation path
|
||||
- canonical index writes are not direct plain writes from multiple locations
|
||||
|
||||
- [ ] **Step 2: Run index-related tests and observe current behavior**
|
||||
|
||||
Run: `python -m pytest tests/test_asset_index.py tests/test_asset_index_integration.py tests/test_repair.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Implement one shared read-modify-write primitive for canonical index updates**
|
||||
|
||||
The primitive must own: lock acquisition, fresh read, mutation callback/application, atomic write, and replace.
|
||||
|
||||
- [ ] **Step 4: Migrate incremental refresh and repair paths to the shared primitive**
|
||||
|
||||
Remove direct `write_text` / `write_json` style index overwrites from these paths.
|
||||
|
||||
- [ ] **Step 5: Re-run index-related tests**
|
||||
|
||||
Run: `python -m pytest tests/test_asset_index.py tests/test_asset_index_integration.py tests/test_repair.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 6: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: one sanctioned mutation path owns canonical index edits.
|
||||
|
||||
### Task A3: Redefine `sync` Final-State Truth
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/services/sync_service.py`
|
||||
- Modify: `paperforge/worker/asset_index.py`
|
||||
- Modify: `paperforge/memory/refresh.py`
|
||||
- Test: `tests/unit/memory/test_refresh.py`
|
||||
- Test: `tests/e2e/test_sync_pipeline.py`
|
||||
- Test: `tests/test_asset_index_integration.py`
|
||||
|
||||
- [ ] **Step 1: Add or extend tests for cleaned end-state semantics**
|
||||
|
||||
Assertions should prove that after `sync` completes, deleted/orphaned notes do not remain in canonical index or refreshed DB state.
|
||||
|
||||
- [ ] **Step 2: Run the focused sync/state tests**
|
||||
|
||||
Run: `python -m pytest tests/unit/memory/test_refresh.py tests/e2e/test_sync_pipeline.py tests/test_asset_index_integration.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Adjust sync ordering or post-clean rebuild logic**
|
||||
|
||||
Choose the minimal change that guarantees published final truth is post-clean, not intermediate.
|
||||
|
||||
- [ ] **Step 4: Verify DB refresh semantics match the new sync contract**
|
||||
|
||||
Update `paperforge/memory/refresh.py` only as needed to avoid stale DB-visible papers after cleanup.
|
||||
|
||||
- [ ] **Step 5: Re-run focused sync/state tests**
|
||||
|
||||
Run: `python -m pytest tests/unit/memory/test_refresh.py tests/e2e/test_sync_pipeline.py tests/test_asset_index_integration.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 6: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: sync success means cleaned end-state truth everywhere.
|
||||
|
||||
### Task A4: Make Plugin Runtime Paths Configuration-Aware
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/plugin/main.js`
|
||||
- Modify: `paperforge/config.py` only if a minimal shared contract helper is needed
|
||||
- Test: `paperforge/plugin/tests/runtime.test.mjs`
|
||||
- Test: `paperforge/plugin/tests/commands.test.mjs`
|
||||
- Test: `tests/test_config.py`
|
||||
|
||||
- [ ] **Step 1: Add failing tests for non-default path layouts and fallback behavior**
|
||||
|
||||
Cover:
|
||||
- runtime snapshot lookup under non-default `system_dir`
|
||||
- missing `paperforge.json`
|
||||
- corrupt `paperforge.json`
|
||||
- fallback to documented defaults
|
||||
- recoverable warning behavior instead of silent mixed-path construction
|
||||
|
||||
- [ ] **Step 2: Run focused plugin/runtime tests**
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/runtime.test.mjs tests/commands.test.mjs`
|
||||
|
||||
- [ ] **Step 3: Introduce one config-aware path resolution flow inside plugin runtime code**
|
||||
|
||||
Reuse existing `paperforge.json` reading patterns where possible. Do not expand into a broader refactor.
|
||||
|
||||
- [ ] **Step 4: Replace hardcoded `System/PaperForge/...` runtime file assumptions**
|
||||
|
||||
Touch only runtime bootstrap, snapshot reads, export polling, and OCR polling code paths.
|
||||
|
||||
- [ ] **Step 5: Re-run plugin/runtime tests plus targeted Python config tests**
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/runtime.test.mjs tests/commands.test.mjs`
|
||||
|
||||
Run: `python -m pytest tests/test_config.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 6: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: plugin runtime paths honor configured directory names and handle missing/corrupt config with explicit fallback behavior.
|
||||
|
||||
### Task A5: Tighten Runtime Lifecycle Behavior
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/commands/embed.py`
|
||||
- Modify: `paperforge/commands/runtime_health.py`
|
||||
- Modify: `paperforge/plugin/main.js`
|
||||
- Test: `tests/unit/commands/test_embed.py`
|
||||
- Test: `tests/unit/commands/test_runtime_health.py`
|
||||
- Test: `paperforge/plugin/tests/errors.test.mjs`
|
||||
|
||||
- [ ] **Step 1: Add failing tests for `embed stop` and first-launch snapshot bootstrap**
|
||||
|
||||
Cover:
|
||||
- missing `os` import / real stop signal path
|
||||
- truthful idle vs stopping result
|
||||
- startup bootstrap producing files the plugin actually checks
|
||||
|
||||
- [ ] **Step 2: Run focused lifecycle tests**
|
||||
|
||||
Run: `python -m pytest tests/unit/commands/test_embed.py tests/unit/commands/test_runtime_health.py -v --tb=short`
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/errors.test.mjs tests/runtime.test.mjs`
|
||||
|
||||
- [ ] **Step 3: Fix stop behavior and bootstrap contract**
|
||||
|
||||
Implement minimal truthful behavior. If stop is best-effort, make the result explicit instead of pretending success.
|
||||
|
||||
- [ ] **Step 4: Restrict startup auto-update behavior to the approved safer model**
|
||||
|
||||
Adjust plugin startup so runtime mutation on load is no longer eager and opaque.
|
||||
|
||||
- [ ] **Step 5: Re-run focused lifecycle tests**
|
||||
|
||||
Run: `python -m pytest tests/unit/commands/test_embed.py tests/unit/commands/test_runtime_health.py -v --tb=short`
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/errors.test.mjs tests/runtime.test.mjs`
|
||||
|
||||
- [ ] **Step 6: Run Package A regression slice**
|
||||
|
||||
Run: `python -m pytest tests/unit/commands/test_embed.py tests/unit/commands/test_runtime_health.py tests/unit/memory/test_refresh.py tests/test_asset_index.py tests/test_asset_index_integration.py tests/test_repair.py tests/e2e/test_sync_pipeline.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 7: Record Package A milestone without committing unless user requests it**
|
||||
|
||||
Expected state: runtime, path, and final-state contracts are hardened.
|
||||
|
||||
---
|
||||
|
||||
## Package B: Command & Skill Truth Alignment
|
||||
|
||||
### Task B1: Freeze the Live CLI Surface as Reference
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/cli.py` only if help text or surface clarification is required
|
||||
- Modify: `paperforge/commands/paper_context.py` if surfaced contract is wrong
|
||||
- Modify: `paperforge/commands/paper_status.py` if surfaced contract is wrong
|
||||
- Modify: `paperforge/commands/search.py` if surfaced contract is wrong
|
||||
- Modify: `paperforge/commands/reading_log.py` if surfaced contract is wrong
|
||||
- Modify: `paperforge/commands/project_log.py` if surfaced contract is wrong
|
||||
- Modify: `paperforge/commands/agent_context.py`
|
||||
- Test: `tests/cli/test_json_contracts.py`
|
||||
- Test: `tests/test_command_docs.py`
|
||||
|
||||
- [ ] **Step 1: Add or extend tests that assert command reference truth comes from the live CLI surface**
|
||||
|
||||
Cover at least command names, supported flags, and any surfaced usage strings emitted to agents.
|
||||
|
||||
- [ ] **Step 2: Run focused CLI/doc tests**
|
||||
|
||||
Run: `python -m pytest tests/cli/test_json_contracts.py tests/test_command_docs.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Minimize CLI ambiguity where necessary**
|
||||
|
||||
Only fix help/usage or agent-context surfaced command metadata if current output itself is stale or misleading.
|
||||
|
||||
- [ ] **Step 3a: Audit whether drift originates in docs only or in command outputs too**
|
||||
|
||||
For each audited mismatch, record whether the truth must be fixed in docs/workflows or in surfaced command output/help text.
|
||||
|
||||
- [ ] **Step 4: Re-run focused CLI/doc tests**
|
||||
|
||||
Run: `python -m pytest tests/cli/test_json_contracts.py tests/test_command_docs.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 5: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: Package B has a stable CLI reference baseline.
|
||||
|
||||
### Task B2: Define Explicit Slash-Route Ownership
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/skills/paperforge/SKILL.md`
|
||||
- Test: `tests/test_command_docs.py`
|
||||
|
||||
- [ ] **Step 1: Add failing assertions or fixtures for slash-command ownership mapping**
|
||||
|
||||
Cover `/pf-sync`, `/pf-ocr`, `/pf-status`, `/pf-deep`, `/pf-paper`, plus unknown-command fallback behavior if testable through doc fixtures.
|
||||
|
||||
- [ ] **Step 2: Update compound skill routing to separate mechanical from cognitive routes**
|
||||
|
||||
Do not add new workflows unless needed. Prefer a clear routing table and explicit unsupported/clarification behavior.
|
||||
|
||||
- [ ] **Step 3: Re-run focused route/doc tests**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 4: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: slash routes have one owner each.
|
||||
|
||||
### Task B3: Repair Workflow Command Accuracy
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/skills/paperforge/workflows/paper-search.md`
|
||||
- Modify: `paperforge/skills/paperforge/workflows/paper-qa.md`
|
||||
- Modify: `paperforge/skills/paperforge/workflows/deep-reading.md`
|
||||
- Modify: `paperforge/skills/paperforge/workflows/reading-log.md`
|
||||
- Modify: `paperforge/skills/paperforge/workflows/project-log.md`
|
||||
- Modify: `paperforge/skills/paperforge/workflows/project-engineering.md`
|
||||
- Test: `tests/test_command_docs.py`
|
||||
|
||||
- [ ] **Step 1: Add failing doc-validation cases for known stale examples**
|
||||
|
||||
Cover the audited issues:
|
||||
- `paper-context` misuse for non-key identifiers
|
||||
- stale lifecycle enum values
|
||||
- invalid `--vault` on `pf_deep.py postprocess-pass2`
|
||||
- stale log render paths
|
||||
|
||||
- [ ] **Step 2: Run focused doc-validation tests**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Rewrite workflow commands to reflect the live CLI exactly**
|
||||
|
||||
Use `paper-status` or `search` first where identifier resolution is needed. Remove or replace stale flags and values. Fix stale output-path language.
|
||||
|
||||
- [ ] **Step 4: Re-run focused doc-validation tests**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 5: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: workflow markdown no longer teaches invalid commands.
|
||||
|
||||
### Task B4: Add Anti-Drift Validation for CLI vs Skill/Docs
|
||||
|
||||
**Files:**
|
||||
- Modify: `tests/test_command_docs.py`
|
||||
- Modify: `tests/cli/test_json_contracts.py`
|
||||
- Modify: `.github/workflows/ci.yml` only if coverage needs to include the validation target
|
||||
|
||||
- [ ] **Step 1: Create a repeatable validation mechanism**
|
||||
|
||||
Prefer fixture- or parser-based checks that detect nonexistent commands/flags in workflow docs and `agent-context` examples.
|
||||
|
||||
- [ ] **Step 2: Run local validation tests**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py tests/cli/test_json_contracts.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Ensure CI executes the validation path**
|
||||
|
||||
Add minimal workflow coverage only if current CI would miss it.
|
||||
|
||||
- [ ] **Step 4: Re-run the same validation tests**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py tests/cli/test_json_contracts.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 5: Run Package B regression slice**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py tests/cli/test_json_contracts.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 6: Record Package B milestone without committing unless user requests it**
|
||||
|
||||
Expected state: doc/skill command drift is machine-detectable.
|
||||
|
||||
### Task B5: Verify Live Slash-Route Ownership and Fallback Behavior
|
||||
|
||||
**Files:**
|
||||
- Reference: `paperforge/skills/paperforge/SKILL.md`
|
||||
- Reference: `paperforge/skills/paperforge/workflows/*.md`
|
||||
- Reference: `paperforge/cli.py`
|
||||
|
||||
- [ ] **Step 1: Define the expected owner and behavior for each known slash route**
|
||||
|
||||
Cover `/pf-sync`, `/pf-ocr`, `/pf-status`, `/pf-deep`, `/pf-paper`.
|
||||
|
||||
- [ ] **Step 2: Verify one malformed or unknown slash command path**
|
||||
|
||||
Confirm it triggers deterministic clarification or unsupported-command behavior rather than accidental fallback.
|
||||
|
||||
- [ ] **Step 3: Record expected vs actual behavior in the milestone notes**
|
||||
|
||||
Capture owner, execution path, and fallback result for each checked route.
|
||||
|
||||
- [ ] **Step 4: Treat this verification as a Package B milestone gate**
|
||||
|
||||
Package B is not complete until route ownership and fallback behavior have been manually verified.
|
||||
|
||||
---
|
||||
|
||||
## Package C: Documentation IA Reset
|
||||
|
||||
### Task C1: Create Canonical User Tutorial and Troubleshooting Docs
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/getting-started.md`
|
||||
- Create: `docs/troubleshooting.md`
|
||||
- Modify: `README.md`
|
||||
- Modify: `README.en.md`
|
||||
|
||||
- [ ] **Step 1: Draft `docs/getting-started.md` as the single user happy-path source**
|
||||
|
||||
Cover install-after-check, BBT export, first sync, OCR when needed, `/pf-deep`, `/pf-paper`, and the shortest normal path.
|
||||
|
||||
- [ ] **Step 2: Draft `docs/troubleshooting.md` for failure recovery only**
|
||||
|
||||
Move user-facing troubleshooting guidance out of README and AGENTS.
|
||||
|
||||
- [ ] **Step 3: Update README links to the new docs only after both docs exist**
|
||||
|
||||
Ensure no duplicate full workflow remains in README.
|
||||
|
||||
- [ ] **Step 4: Manually verify tutorial ownership and link integrity**
|
||||
|
||||
Expected: only one canonical workflow narrative remains.
|
||||
|
||||
- [ ] **Step 5: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: user docs are split into entry page + tutorial + troubleshooting.
|
||||
|
||||
### Task C2: Reshape README into Entry Pages
|
||||
|
||||
**Files:**
|
||||
- Modify: `README.md`
|
||||
- Modify: `README.en.md`
|
||||
|
||||
- [ ] **Step 1: Create a target outline for both READMEs**
|
||||
|
||||
The outline must be limited to overview, install options, quickstart, and doc map.
|
||||
|
||||
- [ ] **Step 2: Rewrite `README.md` as a navigation-first entry page**
|
||||
|
||||
Remove full tutorial, deep troubleshooting, and maintainer detail.
|
||||
|
||||
- [ ] **Step 3: Rewrite `README.en.md` to match the same structure**
|
||||
|
||||
Keep scope equivalent even if wording differs.
|
||||
|
||||
- [ ] **Step 4: Verify both README files point to the new canonical docs**
|
||||
|
||||
Manually check links and scope boundaries.
|
||||
|
||||
- [ ] **Step 5: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: README is no longer the entire manual.
|
||||
|
||||
### Task C3: Convert `AGENTS.md` into Agent-Only Guidance
|
||||
|
||||
**Files:**
|
||||
- Modify: `AGENTS.md`
|
||||
- Modify: `docs/getting-started.md`
|
||||
- Modify: `docs/troubleshooting.md`
|
||||
|
||||
- [ ] **Step 1: Strip end-user tutorial, FAQ, and user update content from `AGENTS.md`**
|
||||
|
||||
Keep repo operating rules, architecture boundaries, source-of-truth guidance, and doc map for agents.
|
||||
|
||||
- [ ] **Step 2: Move any removed user guidance into the correct user docs**
|
||||
|
||||
Do not silently drop content that still matters.
|
||||
|
||||
- [ ] **Step 3: Add a concise doc map in `AGENTS.md`**
|
||||
|
||||
Point agents to README, getting-started, troubleshooting, commands, architecture, and maintainer docs by audience.
|
||||
|
||||
- [ ] **Step 4: Manually verify that a normal user no longer needs `AGENTS.md`**
|
||||
|
||||
- [ ] **Step 5: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: `AGENTS.md` is agent-only.
|
||||
|
||||
### Task C4: Purify Command Reference and Establish Maintainer Ownership
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/COMMANDS.md`
|
||||
- Modify: `docs/ARCHITECTURE.md`
|
||||
- Create: `docs/maintainer-guide.md`
|
||||
|
||||
- [ ] **Step 1: Rewrite `docs/COMMANDS.md` into pure reference form**
|
||||
|
||||
Keep command purpose, prerequisites, key flags, examples. Remove tutorial and stale platform roadmap language.
|
||||
|
||||
- [ ] **Step 2: Refresh `docs/ARCHITECTURE.md` to current architecture reality or reduce it to a stable maintainer overview**
|
||||
|
||||
Remove stale version framing and outdated file ownership claims.
|
||||
|
||||
- [ ] **Step 3: Create `docs/maintainer-guide.md`**
|
||||
|
||||
Own release, testing, versioning, migration links, and architecture-navigation concerns here.
|
||||
|
||||
- [ ] **Step 4: Verify link integrity and audience split manually**
|
||||
|
||||
- [ ] **Step 5: Run doc-validation tests if they cover links or command examples**
|
||||
|
||||
Run: `python -m pytest tests/test_command_docs.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 6: Record Package C milestone without committing unless user requests it**
|
||||
|
||||
Expected state: doc IA is explicit by file ownership.
|
||||
|
||||
---
|
||||
|
||||
## Package D: Progressive Disclosure UX
|
||||
|
||||
### Task D1: Stage the Setup Wizard Around First Sync
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/plugin/main.js`
|
||||
- Test: `paperforge/plugin/tests/runtime.test.mjs`
|
||||
- Test: `tests/journey/test_onboarding.py`
|
||||
|
||||
- [ ] **Step 1: Define and encode first-sync-only required fields**
|
||||
|
||||
Separate mandatory-for-first-sync from later-stage OCR/agent/vector configuration.
|
||||
|
||||
- [ ] **Step 2: Add or update onboarding tests to reflect staged requirements**
|
||||
|
||||
Run: `python -m pytest tests/journey/test_onboarding.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 3: Implement staged setup gating in the plugin**
|
||||
|
||||
Do not redesign the wizard; only change requirement timing and copy.
|
||||
|
||||
- [ ] **Step 5: Re-run onboarding and plugin tests**
|
||||
|
||||
Run: `python -m pytest tests/journey/test_onboarding.py -v --tb=short`
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/runtime.test.mjs`
|
||||
|
||||
- [ ] **Step 6: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: first sync is no longer blocked by later-stage features.
|
||||
|
||||
### Task D2: Fix Persistent Core Action Visibility
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/plugin/main.js`
|
||||
- Test: `paperforge/plugin/tests/commands.test.mjs`
|
||||
|
||||
- [ ] **Step 1: Identify the current primary action rendering path for the Home/Global surface**
|
||||
|
||||
Map where Sync, OCR, Status, Doctor, Repair, and Open Library are conditionally shown.
|
||||
|
||||
- [ ] **Step 2: Add or extend tests for persistent primary action visibility**
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/commands.test.mjs`
|
||||
|
||||
- [ ] **Step 3: Consolidate core actions into one persistent primary action group**
|
||||
|
||||
Do not redesign the information model; only normalize placement.
|
||||
|
||||
- [ ] **Step 4: Re-run plugin action tests**
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/commands.test.mjs`
|
||||
|
||||
- [ ] **Step 5: Record checkpoint without committing unless user requests it**
|
||||
|
||||
Expected state: users no longer have to guess where core mechanical actions live.
|
||||
|
||||
### Task D3: Collapse Advanced Controls and Normalize User-Facing Copy
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/plugin/main.js`
|
||||
- Test: `paperforge/plugin/tests/errors.test.mjs`
|
||||
|
||||
- [ ] **Step 1: Freeze the glossary chosen by Package C**
|
||||
|
||||
List the canonical terms for Dashboard/Home, Runtime update action, Paper view naming, and other repeated surfaces.
|
||||
|
||||
- [ ] **Step 2: Hide advanced memory/vector/runtime controls behind one default-collapsed section**
|
||||
|
||||
Retain access for power users.
|
||||
|
||||
- [ ] **Step 3: Rewrite user-facing errors to lead with next action**
|
||||
|
||||
Keep technical details available secondarily.
|
||||
|
||||
- [ ] **Step 4: Sweep docs and UI copy for glossary compliance**
|
||||
|
||||
Touch only user-facing strings inside plugin UI. Documentation glossary adoption belongs to Package C.
|
||||
|
||||
- [ ] **Step 5: Re-run focused plugin error tests and perform a manual first-run walkthrough**
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/errors.test.mjs`
|
||||
|
||||
- [ ] **Step 6: Record Package D milestone without committing unless user requests it**
|
||||
|
||||
Expected state: complexity is reduced by staging, hiding, and consistent terms rather than by a rewrite.
|
||||
|
||||
---
|
||||
|
||||
## Final Verification
|
||||
|
||||
- [ ] **Step 1: Run the Python contract/doc regression slice**
|
||||
|
||||
Run: `python -m pytest tests/unit/commands/test_embed.py tests/unit/commands/test_runtime_health.py tests/unit/memory/test_refresh.py tests/test_asset_index.py tests/test_asset_index_integration.py tests/test_repair.py tests/cli/test_json_contracts.py tests/test_command_docs.py tests/journey/test_onboarding.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 2: Run plugin regression slice**
|
||||
|
||||
Run: `cd paperforge/plugin && npx vitest run tests/runtime.test.mjs tests/commands.test.mjs tests/errors.test.mjs tests/vector-ready.test.mjs`
|
||||
|
||||
- [ ] **Step 3: Run broader repo checks if time and scope allow**
|
||||
|
||||
Run: `python -m pytest tests/e2e/test_sync_pipeline.py tests/e2e/test_status_doctor_repair.py -v --tb=short`
|
||||
|
||||
- [ ] **Step 4: Verify docs manually by audience**
|
||||
|
||||
Check:
|
||||
- user can navigate from README to getting-started without reading AGENTS
|
||||
- agent can learn repo rules from AGENTS without user workflow noise
|
||||
- maintainer can find release/testing/version docs from `docs/maintainer-guide.md`
|
||||
|
||||
- [ ] **Step 5: Prepare release notes by milestone, but do not create commits unless the user explicitly asks**
|
||||
325
docs/superpowers/plans/2026-05-18-discussion-md-truth-source.md
Normal file
325
docs/superpowers/plans/2026-05-18-discussion-md-truth-source.md
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
# Discussion MD as Truth Source — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Make `discussion.md` the sole truth source, remove `discussion.json`, render rich markdown in plugin card via `MarkdownRenderer.render()`.
|
||||
|
||||
**Architecture:** `discussion.py` stops writing JSON, drops `_escape_md()`, changes lock file. Plugin card reads `.md` directly and uses Obsidian's built-in `MarkdownRenderer.render()` for native rendering.
|
||||
|
||||
**Tech Stack:** Python stdlib, Obsidian `MarkdownRenderer` API
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `discussion.py` — Remove JSON, remove escaping, change lock
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/worker/discussion.py`
|
||||
|
||||
- [ ] **Step 1: Remove `_escape_md()` and related cruft**
|
||||
|
||||
Delete lines 40-41 (`_MD_SPECIAL_CHARS` regex + comment):
|
||||
```python
|
||||
_MD_SPECIAL_CHARS = re.compile(r"([*#\[\]_`])")
|
||||
"""Regex for markdown special characters that must be escaped in QA text fields."""
|
||||
```
|
||||
|
||||
Delete lines 59-66 (`_escape_md()` function):
|
||||
```python
|
||||
def _escape_md(text: str) -> str:
|
||||
...
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Update `_build_md_session()` to not escape**
|
||||
|
||||
In lines 186-187, change:
|
||||
```python
|
||||
lines.append(f"**问题:** {_escape_md(qa['question'])}")
|
||||
lines.append(f"**解答:** {_escape_md(qa['answer'])}\n")
|
||||
```
|
||||
to:
|
||||
```python
|
||||
lines.append(f"**问题:** {qa['question']}")
|
||||
lines.append(f"**解答:** {qa['answer']}\n")
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Remove `_atomic_write_json()` function**
|
||||
|
||||
Delete lines 145-171 (`_atomic_write_json()` function).
|
||||
|
||||
- [ ] **Step 4: Update `record_session()` — remove JSON, change lock**
|
||||
|
||||
In `record_session()`:
|
||||
- Remove line 287: `json_path = ai_dir / "discussion.json"`
|
||||
- Change line 300: `.json.lock` → `.md.lock`
|
||||
- Remove JSON reading/writing block (lines 304-319): the `# Write discussion.json` block
|
||||
- Remove `json_path` from return value (line 343)
|
||||
|
||||
The rewrite looks like:
|
||||
```python
|
||||
md_path = ai_dir / "discussion.md"
|
||||
|
||||
session = _build_session(agent, model, zotero_key, paper_title, domain, qa_pairs)
|
||||
|
||||
lock_path = md_path.with_suffix(".md.lock")
|
||||
lock = filelock.FileLock(lock_path, timeout=LOCK_TIMEOUT)
|
||||
try:
|
||||
with lock:
|
||||
existing_md = ""
|
||||
if md_path.exists():
|
||||
try:
|
||||
existing_md = md_path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
logger.warning("Could not read existing discussion.md: %s", exc)
|
||||
existing_md = ""
|
||||
|
||||
header = _build_md_header(paper_title)
|
||||
session_md = _build_md_session(session)
|
||||
content = _md_content(existing_md, header, session_md)
|
||||
_atomic_write_md(md_path, content)
|
||||
except filelock.Timeout:
|
||||
...
|
||||
return {"status": "error", "message": "Concurrent access conflict. Please try again."}
|
||||
except Exception as exc:
|
||||
...
|
||||
return {"status": "error", "message": f"Failed to write discussion file: {exc}"}
|
||||
|
||||
return {"status": "ok", "md_path": str(md_path)}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Update docstrings**
|
||||
|
||||
In module docstring (lines 3-4), change `JSON (canonical)` → `Markdown (canonical)`:
|
||||
```python
|
||||
"""Discussion recorder — writes structured AI-paper Q&A into ai/ workspace directory.
|
||||
|
||||
Atomic append-only writes for Markdown (canonical). stdlib only.
|
||||
"""
|
||||
```
|
||||
|
||||
In `record_session()` docstring (lines 235-237), change:
|
||||
```python
|
||||
"""Record an AI-paper discussion session.
|
||||
|
||||
Creates or appends to ai/discussion.md (canonical) with atomic writes.
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Remove unused `import re`**
|
||||
|
||||
Remove `import re` (line 20) — no longer needed (no `_MD_SPECIAL_CHARS`). Keep `import json`.
|
||||
|
||||
- [ ] **Step 8: Run existing tests to see what breaks**
|
||||
Run: `python -m pytest tests/test_discussion.py -v --tb=short`
|
||||
Expected: several failures (JSON assertions, test_utc_timestamp, escape tests, etc.)
|
||||
|
||||
- [ ] **Step 9: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(discussion): remove JSON output and markdown escaping"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Plugin card — Read `.md`, render with `MarkdownRenderer`
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/plugin/main.js`
|
||||
|
||||
- [ ] **Step 1: Rewrite `_renderRecentDiscussionCard()`**
|
||||
|
||||
Replace the current implementation (lines 1959-2031) with:
|
||||
|
||||
```js
|
||||
/* ── Recent Discussion Card: read ai/discussion.md ── */
|
||||
_renderRecentDiscussionCard(container, entry) {
|
||||
const card = container.createEl('div', { cls: 'paperforge-discussion-card' });
|
||||
card.style.display = 'none';
|
||||
|
||||
if (!entry.note_path) return;
|
||||
const lastSlash = entry.note_path.lastIndexOf('/');
|
||||
const wsDir = lastSlash !== -1 ? entry.note_path.substring(0, lastSlash) : '.';
|
||||
const mdPath = wsDir + '/ai/discussion.md';
|
||||
|
||||
this.app.vault.adapter.exists(mdPath).then((exists) => {
|
||||
if (!exists) return;
|
||||
return this.app.vault.adapter.read(mdPath);
|
||||
}).then((raw) => {
|
||||
if (!raw) return;
|
||||
|
||||
const pairs = this._parseDiscussionMD(raw);
|
||||
if (!pairs || pairs.length === 0) return;
|
||||
|
||||
card.style.display = 'block';
|
||||
const header = card.createEl('div', { cls: 'paperforge-discussion-header' });
|
||||
header.createEl('span', { cls: 'paperforge-discussion-title', text: '\u6700\u8fd1\u8ba8\u8bba' });
|
||||
|
||||
for (const qa of pairs) {
|
||||
const item = card.createEl('div', { cls: 'paperforge-discussion-item' });
|
||||
const qEl = item.createEl('div', { cls: 'paperforge-discussion-q' });
|
||||
await MarkdownRenderer.render(this.app, '**\u63d0\u95ee\uff1a**' + qa.question, qEl, mdPath, this);
|
||||
|
||||
const aEl = item.createEl('div', { cls: 'paperforge-discussion-a' });
|
||||
if (qa.answer && qa.answer.length > 500) {
|
||||
aEl.style.maxHeight = '200px';
|
||||
aEl.style.overflow = 'hidden';
|
||||
const toggle = item.createEl('button', { cls: 'paperforge-expand-btn', text: '\u5c55\u5f00\u66f4\u591a \u25bd' });
|
||||
let expanded = false;
|
||||
toggle.addEventListener('click', () => {
|
||||
expanded = !expanded;
|
||||
aEl.style.maxHeight = expanded ? '' : '200px';
|
||||
toggle.setText(expanded ? '\u6536\u8d77 \u25b3' : '\u5c55\u5f00\u66f4\u591a \u25bd');
|
||||
});
|
||||
}
|
||||
await MarkdownRenderer.render(this.app, qa.answer || '', aEl, mdPath, this);
|
||||
}
|
||||
|
||||
// "查看全部" link
|
||||
const viewAll = card.createEl('a', { cls: 'paperforge-discussion-viewall', text: '\u67e5\u770b\u5168\u90e8\u8ba8\u8bba \u2192' });
|
||||
viewAll.addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
const discFile = this.app.vault.getAbstractFileByPath(mdPath);
|
||||
if (discFile) {
|
||||
this.app.workspace.openLinkText(mdPath, '');
|
||||
} else {
|
||||
new Notice('\u8ba8\u8bba\u6587\u4ef6\u5c1a\u672a\u751f\u6210');
|
||||
}
|
||||
});
|
||||
}).catch((e) => {
|
||||
console.error('PaperForge: discussion.md read error', mdPath, e.message);
|
||||
});
|
||||
}
|
||||
|
||||
_parseDiscussionMD(content) {
|
||||
const sessions = content.split(/\n## /).slice(1);
|
||||
if (sessions.length === 0) return null;
|
||||
const lastSession = sessions[sessions.length - 1];
|
||||
const pairs = [];
|
||||
const qaBlocks = lastSession.split(/\*\*\u95ee\u9898:\*\*/).slice(1);
|
||||
for (const block of qaBlocks) {
|
||||
const answerMatch = block.match(/\*\*\u89e3\u7b54:\*\*/);
|
||||
if (!answerMatch) continue;
|
||||
const question = block.substring(0, answerMatch.index).trim();
|
||||
const answer = block.substring(answerMatch.index + '**解答:**'.length).trim();
|
||||
pairs.push({ question, answer });
|
||||
}
|
||||
return pairs.slice(-3);
|
||||
}
|
||||
```
|
||||
|
||||
**Note about `MarkdownRenderer`:** main.js line 1 already uses `const { Plugin, /* etc */ } = require('obsidian');`. Just add `MarkdownRenderer` to that destructuring:
|
||||
```js
|
||||
const { Plugin, MarkdownRenderer, /* ... */ } = require('obsidian');
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run existing tests**
|
||||
Run: `python -m pytest tests/ -v --tb=short`
|
||||
Expected: all Python tests pass (JS change is untestable via pytest)
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(plugin): render discussion card from .md with MarkdownRenderer"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Update `paper-qa.md` description
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/skills/paperforge/workflows/paper-qa.md`
|
||||
|
||||
- [ ] **Step 1: Update line 8 text**
|
||||
|
||||
Change:
|
||||
```markdown
|
||||
每次问答记录到 `discussion.json`（Dashboard 可见）。
|
||||
```
|
||||
to:
|
||||
```markdown
|
||||
每次问答记录到 `discussion.md`（Dashboard 可见）。
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "docs(paper-qa): update discussion.json -> discussion.md"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Update tests
|
||||
|
||||
**Files:**
|
||||
- Modify: `tests/test_discussion.py`
|
||||
|
||||
- [ ] **Step 1: Rewrite tests to match new behavior**
|
||||
|
||||
- Rename `test_create_both_files` → `test_create_md_file`: remove JSON assertions, verify .md content is rich (no escaping)
|
||||
- `test_append_second_session`: change to use `.md` lock
|
||||
- `test_markdown_escaping`: **reverse** — verify `**bold**` is preserved unescaped
|
||||
- `test_markdown_escaping_cjk`: same
|
||||
- `test_file_lock`: change `.json.lock` → `.md.lock`
|
||||
- `test_lock_timeout_returns_error`: same
|
||||
- `test_cli_invocation`: no longer checks `json_path`
|
||||
- `test_cjk_encoding`: replace `result["json_path"]` with `result["md_path"]`
|
||||
- `test_atomic_write_no_partial`: replace `result["json_path"]` with `result["md_path"]`, remove JSON assertions
|
||||
- Remove `test_utc_timestamp` entirely
|
||||
- Remove `import json` if no longer needed in test
|
||||
|
||||
Key test changes:
|
||||
```python
|
||||
def test_create_md_file(self, tmp_path: Path) -> None:
|
||||
"""Test: Creates ai/discussion.md with rich unescaped markdown."""
|
||||
vault = _create_minimal_vault(tmp_path)
|
||||
result = record_session(...)
|
||||
assert result["status"] == "ok"
|
||||
assert "json_path" not in result
|
||||
md_path = Path(result["md_path"])
|
||||
assert md_path.exists()
|
||||
md_content = md_path.read_text(encoding="utf-8")
|
||||
assert "# AI Discussion Record:" in md_content
|
||||
assert "**问题:**" in md_content
|
||||
assert "**解答:**" in md_content
|
||||
|
||||
def test_markdown_preserved_unescaped(self, tmp_path: Path) -> None:
|
||||
"""Test: **bold** is NOT escaped to \*\*bold\*\*."""
|
||||
vault = _create_minimal_vault(tmp_path)
|
||||
qa = [{"question": "What does **bold** mean?",
|
||||
"answer": "Use **bold** and `code`",
|
||||
"source": "user_question",
|
||||
"timestamp": "2026-05-06T12:00:00+00:00"}]
|
||||
result = record_session(...)
|
||||
md = Path(result["md_path"]).read_text(encoding="utf-8")
|
||||
assert "**bold**" in md # NOT \*\*bold\*\*
|
||||
assert "\\*" not in md # no escaping
|
||||
|
||||
def test_file_lock_uses_md_lock(self, tmp_path: Path) -> None:
|
||||
"""Lock file is .md.lock not .json.lock."""
|
||||
vault = _create_minimal_vault(tmp_path)
|
||||
result = record_session(...)
|
||||
md_path = Path(result["md_path"])
|
||||
lock_path = md_path.with_suffix(".md.lock")
|
||||
assert not lock_path.exists()
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests**
|
||||
Run: `python -m pytest tests/test_discussion.py -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "test(discussion): update tests for MD-only truth source"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Full suite + lint
|
||||
|
||||
- [ ] **Step 1: Run all tests**
|
||||
Run: `python -m pytest tests/ -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 2: Run lint**
|
||||
Run: `ruff check --fix paperforge/ && ruff format paperforge/`
|
||||
Expected: clean
|
||||
|
||||
- [ ] **Step 3: Final commit if lint fixes applied**
|
||||
```bash
|
||||
git add -A && git commit -m "style: lint after discussion MD-truth-source changes"
|
||||
```
|
||||
1156
docs/superpowers/plans/2026-05-18-memory-layer-simplify.md
Normal file
1156
docs/superpowers/plans/2026-05-18-memory-layer-simplify.md
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,653 @@
|
|||
# Vector Status + Health + Auto-Embed — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add per-paper `vector_status` column to `paperforge.db`, make `_check_vector()` strict, fix resume to use SQLite, and wire auto-embed status writing after OCR.
|
||||
|
||||
**Architecture:** Schema v3 adds `vector_status TEXT` column. Embed builder writes status after each paper. `_check_vector()` queries coverage ratio. Resume filters via SQL instead of ChromaDB per-paper lookups. Auto-embed after OCR writes status back to DB.
|
||||
|
||||
**Tech Stack:** SQLite (same `paperforge.db`), Python, ChromaDB, OpenAI embeddings API
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Schema v3 — Add vector_status column
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/schema.py:9` — `CURRENT_SCHEMA_VERSION = 3`
|
||||
- Modify: `paperforge/memory/_columns.py:6-15` — add `"vector_status"` to `PAPER_COLUMNS`
|
||||
- Modify: `paperforge/memory/builder.py:130-279` — preserve `vector_status` across rebuild
|
||||
- Test: `tests/unit/memory/test_schema.py` — verify `vector_status` column exists after schema create
|
||||
|
||||
- [ ] **Step 1: Write schema + columns changes**
|
||||
|
||||
In `paperforge/memory/schema.py:9`, change:
|
||||
```python
|
||||
CURRENT_SCHEMA_VERSION = 3 # Bump from 2 for vector_status column
|
||||
```
|
||||
|
||||
In `paperforge/memory/_columns.py`, add `"vector_status"` to `PAPER_COLUMNS` after `"ocr_job_id"`:
|
||||
```python
|
||||
"ocr_job_id", "vector_status", "impact_factor",
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Preserve vector_status in memory builder**
|
||||
|
||||
In `paperforge/memory/builder.py`, before the DELETE of papers (line 166), save a map of existing `vector_status` values:
|
||||
|
||||
```python
|
||||
# Before DELETE, save existing vector_status
|
||||
existing_status = {}
|
||||
try:
|
||||
rows = conn.execute("SELECT zotero_key, vector_status FROM papers WHERE vector_status != ''").fetchall()
|
||||
existing_status = {row["zotero_key"]: row["vector_status"] for row in rows}
|
||||
except Exception:
|
||||
pass # Table or column may not exist yet
|
||||
```
|
||||
|
||||
After the `paper_rows` build (after line 228, before meta upserts), restore saved status:
|
||||
```python
|
||||
# Restore vector_status for papers that had it
|
||||
for row in paper_rows:
|
||||
saved = existing_status.get(row.get("zotero_key", ""))
|
||||
if saved:
|
||||
conn.execute(
|
||||
"UPDATE papers SET vector_status = ? WHERE zotero_key = ?",
|
||||
(saved, row["zotero_key"]),
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Write test for schema v3 vector_status column**
|
||||
|
||||
In `tests/unit/memory/test_schema.py`:
|
||||
```python
|
||||
def test_schema_v3_has_vector_status_column():
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp:
|
||||
db_path = Path(tmp.name)
|
||||
try:
|
||||
conn = get_connection(db_path)
|
||||
ensure_schema(conn)
|
||||
cols = {row["name"] for row in conn.execute("PRAGMA table_info(papers)").fetchall()}
|
||||
assert "vector_status" in cols
|
||||
conn.close()
|
||||
finally:
|
||||
db_path.unlink(missing_ok=True)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
Run: `python -m pytest tests/unit/memory/test_schema.py::test_schema_v3_has_vector_status_column -v --tb=short`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Run all existing tests to confirm no regression**
|
||||
Run: `python -m pytest tests/unit/memory/test_schema.py -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(schema): add vector_status column (v3)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Embed build — Write per-paper vector_status to DB
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/commands/embed.py` — before build loop UPDATE to pending, after each paper UPDATE status, resume filter via SQL
|
||||
- Test: `tests/unit/commands/test_embed.py`
|
||||
|
||||
- [ ] **Step 1: Add DB connection inside run() after vault is available**
|
||||
|
||||
After line 27 (`vault = args.vault_path`), add:
|
||||
```python
|
||||
from paperforge.memory.db import get_connection, get_memory_db_path
|
||||
_db = get_connection(get_memory_db_path(vault))
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Set all OCR-done papers to pending before loop (unless resume)**
|
||||
|
||||
Before the `for entry in done_papers:` loop (before line 166), add:
|
||||
```python
|
||||
if not resume:
|
||||
_db.execute(
|
||||
"UPDATE papers SET vector_status = 'pending' WHERE ocr_status = 'done'"
|
||||
)
|
||||
_db.commit()
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Check for corruption before resume loop**
|
||||
|
||||
After the `if not resume: UPDATE ... pending` block, add corruption warning:
|
||||
```python
|
||||
if resume:
|
||||
embed_status = get_embed_status(vault)
|
||||
if embed_status.get("corrupted", False):
|
||||
import logging
|
||||
logging.getLogger(__name__).warning(
|
||||
"ChromaDB index is corrupted. Resume will attempt to embed missing papers, "
|
||||
"but collection.add() may still fail. Use --force if errors occur."
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Replace resume ChromaDB check with SQLite filter**
|
||||
|
||||
Replace lines 173-184 (the ChromaDB per-paper `collection.get()` block) with:
|
||||
```python
|
||||
if resume:
|
||||
row = _db.execute(
|
||||
"SELECT vector_status FROM papers WHERE zotero_key = ?",
|
||||
(key,)
|
||||
).fetchone()
|
||||
if row and row["vector_status"] == "embedded":
|
||||
papers_skipped += 1
|
||||
continue
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Write vector_status after each paper embed success/failure**
|
||||
|
||||
After `papers_embedded += 1` (line 194), add:
|
||||
```python
|
||||
_db.execute(
|
||||
"UPDATE papers SET vector_status = 'embedded' WHERE zotero_key = ?",
|
||||
(key,)
|
||||
)
|
||||
_db.commit()
|
||||
```
|
||||
|
||||
In the except block (before `return 1` on line 220), add:
|
||||
```python
|
||||
_db.execute(
|
||||
"UPDATE papers SET vector_status = 'failed' WHERE zotero_key = ?",
|
||||
(key,)
|
||||
)
|
||||
_db.commit()
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Close DB at end**
|
||||
|
||||
After the final `write_vector_runtime` call (line 241), add:
|
||||
```python
|
||||
_db.close()
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Write test for resume using SQLite**
|
||||
|
||||
In `tests/unit/commands/test_embed.py`, add:
|
||||
|
||||
```python
|
||||
def test_embed_resume_does_not_call_collection_get(tmp_path):
|
||||
from argparse import Namespace
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir()
|
||||
(vault / "paperforge.json").write_text(
|
||||
'{"vault_config":{"system_dir":"System","resources_dir":"Resources","literature_dir":"Literature","control_dir":"LiteratureControl","base_dir":"Bases","skill_dir":".opencode/skills"}}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
from paperforge.memory.db import get_connection, get_memory_db_path
|
||||
from paperforge.memory.schema import ensure_schema, CURRENT_SCHEMA_VERSION
|
||||
db_path = get_memory_db_path(vault)
|
||||
conn = get_connection(db_path)
|
||||
ensure_schema(conn)
|
||||
conn.execute("INSERT INTO meta (key, value) VALUES ('schema_version', ?)", (str(CURRENT_SCHEMA_VERSION),))
|
||||
conn.commit()
|
||||
|
||||
# Add a paper that's already embedded
|
||||
conn.execute("""INSERT INTO papers (zotero_key, title, ocr_status, vector_status)
|
||||
VALUES (?, ?, ?, ?)""",
|
||||
("EXISTING", "Test Paper", "done", "embedded"))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
with patch("paperforge.commands.embed._preflight_check") as m_pre:
|
||||
m_pre.return_value = {"ok": True}
|
||||
with patch("paperforge.commands.embed.read_index") as m_idx:
|
||||
m_idx.return_value = {
|
||||
"items": [
|
||||
{"zotero_key": "EXISTING", "ocr_status": "done", "fulltext_path": "x.md"},
|
||||
{"zotero_key": "NEWPAPER", "ocr_status": "done", "fulltext_path": "x.md"},
|
||||
]
|
||||
}
|
||||
with patch("paperforge.commands.embed.chunk_fulltext") as m_chunk:
|
||||
m_chunk.return_value = [{"text": "hello", "chunk_index": 0, "section": "", "page_number": 1, "token_estimate": 1}]
|
||||
with patch("paperforge.commands.embed.get_collection") as m_col:
|
||||
# mock collection for embed_paper only
|
||||
m_col.return_value.count.return_value = 0
|
||||
with patch("paperforge.commands.embed.OpenAICompatibleProvider") as m_prov:
|
||||
m_prov.return_value.encode.return_value = [[0.1]]
|
||||
with patch("paperforge.commands.embed.get_embed_status") as m_stat:
|
||||
m_stat.return_value = {"mode": "api", "model": "test"}
|
||||
with patch.object(sys, "stdout", open("NUL", "w") if sys.platform == "win32" else open("/dev/null", "w")):
|
||||
args = Namespace(
|
||||
vault_path=vault,
|
||||
embed_subcommand="build",
|
||||
force=False,
|
||||
resume=True,
|
||||
json=True,
|
||||
)
|
||||
rc = run(args)
|
||||
assert rc == 0
|
||||
# Verify collection.get was never called from resume check
|
||||
# get_collection is called from embed_paper (add) and get_embed_status (count)
|
||||
# but NOT from the resume loop itself
|
||||
# The resume SQL should skip EXISTING, so only NEWPAPER gets embedded
|
||||
# Verify NEWPAPER got vector_status='embedded'
|
||||
conn2 = get_connection(db_path)
|
||||
row = conn2.execute("SELECT vector_status FROM papers WHERE zotero_key='NEWPAPER'").fetchone()
|
||||
conn2.close()
|
||||
assert row["vector_status"] == "embedded"
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Run tests**
|
||||
Run: `python -m pytest tests/unit/commands/test_embed.py -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 9: Run all unit tests**
|
||||
Run: `python -m pytest tests/unit/ -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 10: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(embed): write per-paper vector_status to DB, resume via SQLite"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Strict `_check_vector()` with coverage ratio
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/runtime_health.py:120-153`
|
||||
- Test: `tests/unit/memory/test_runtime_health.py`
|
||||
|
||||
- [ ] **Step 1: Rewrite `_check_vector()`**
|
||||
|
||||
Replace lines 120-153 with:
|
||||
```python
|
||||
def _check_vector(vault: Path) -> dict:
|
||||
from paperforge.embedding import get_vector_db_path, read_vector_build_state
|
||||
from paperforge.embedding.status import get_embed_status
|
||||
|
||||
settings_path = vault / ".obsidian" / "plugins" / "paperforge" / "data.json"
|
||||
vector_enabled = False
|
||||
if settings_path.exists():
|
||||
try:
|
||||
import json
|
||||
data = json.loads(settings_path.read_text(encoding="utf-8"))
|
||||
vector_enabled = bool(data.get("features", {}).get("vector_db", False))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not vector_enabled:
|
||||
return _layer("degraded", ["Vector DB disabled by user"],
|
||||
"Enable vector DB in plugin settings to use semantic search",
|
||||
"")
|
||||
|
||||
build_state = read_vector_build_state(vault)
|
||||
job_status = build_state.get("status", "idle")
|
||||
if job_status == "running":
|
||||
return _layer("degraded", ["Vector build in progress"],
|
||||
"Wait for build to complete",
|
||||
"paperforge embed status --json")
|
||||
if job_status == "failed":
|
||||
return _layer("degraded", [f"Last build failed: {build_state.get('message', '')}"],
|
||||
"Check error and rebuild",
|
||||
"paperforge embed build --resume")
|
||||
|
||||
embed_status = get_embed_status(vault)
|
||||
if embed_status.get("corrupted", False):
|
||||
return _layer("degraded",
|
||||
["ChromaDB index corrupted"],
|
||||
"Rebuild from scratch",
|
||||
"paperforge embed build --force")
|
||||
|
||||
db_path = get_vector_db_path(vault)
|
||||
if not db_path.exists():
|
||||
return _layer("degraded", ["Vector DB not built yet"],
|
||||
"Run embed build",
|
||||
"paperforge embed build --resume")
|
||||
|
||||
# Query coverage from paperforge.db
|
||||
try:
|
||||
from paperforge.memory.db import get_connection, get_memory_db_path
|
||||
mem_db = get_memory_db_path(vault)
|
||||
if not mem_db.exists():
|
||||
return _layer("degraded", ["Memory DB not found; cannot check vector coverage"],
|
||||
"Run paperforge memory build",
|
||||
"paperforge memory build")
|
||||
conn = get_connection(mem_db, read_only=True)
|
||||
total_ocr = conn.execute(
|
||||
"SELECT COUNT(*) FROM papers WHERE ocr_status = 'done'"
|
||||
).fetchone()[0]
|
||||
embedded = conn.execute(
|
||||
"SELECT COUNT(*) FROM papers WHERE ocr_status = 'done' AND vector_status = 'embedded'"
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
return _layer("degraded", [f"Cannot query vector coverage: {e}"],
|
||||
"Check DB integrity", "paperforge doctor")
|
||||
|
||||
if total_ocr == 0:
|
||||
return _layer("degraded", ["No OCR-done papers to embed"],
|
||||
"Run OCR on papers first",
|
||||
"paperforge ocr run")
|
||||
|
||||
if embedded == 0 and db_path.exists():
|
||||
return _layer("degraded", ["Vector DB exists but no papers marked as embedded"],
|
||||
"Rebuild to ensure consistency",
|
||||
"paperforge embed build --resume")
|
||||
|
||||
if embedded == 0:
|
||||
return _layer("degraded", ["No papers embedded yet (0/0 coverage)"],
|
||||
"Run embed build",
|
||||
"paperforge embed build --resume")
|
||||
|
||||
ratio = embedded / total_ocr
|
||||
if ratio < 1.0:
|
||||
evidence = [f"Partial embedding coverage: {embedded}/{total_ocr} ({ratio:.0%})"]
|
||||
return _layer("degraded", evidence,
|
||||
"Run embed build --resume to finish remaining papers",
|
||||
"paperforge embed build --resume")
|
||||
|
||||
return _layer("ok",
|
||||
[f"All {embedded} OCR-done papers embedded ({ratio:.0%} coverage)"])
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write tests for degraded/disabling/partial coverage**
|
||||
|
||||
In `tests/unit/memory/test_runtime_health.py`, add:
|
||||
|
||||
```python
|
||||
def test_check_vector_disabled_returns_degraded(tmp_path):
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir()
|
||||
(vault / "paperforge.json").write_text(
|
||||
json.dumps({"system_dir": "System"}), encoding="utf-8"
|
||||
)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").parent.mkdir(parents=True)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").write_text(
|
||||
json.dumps({"features": {"vector_db": False}}), encoding="utf-8"
|
||||
)
|
||||
result = _check_vector(vault)
|
||||
assert result["status"] == "degraded"
|
||||
assert "disabled" in result["evidence"][0].lower()
|
||||
```
|
||||
|
||||
```python
|
||||
def test_check_vector_corrupted_returns_degraded(tmp_path):
|
||||
from paperforge.embedding import get_vector_db_path
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir(parents=True)
|
||||
(vault / "paperforge.json").write_text(
|
||||
json.dumps({"system_dir": "System"}), encoding="utf-8"
|
||||
)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").parent.mkdir(parents=True)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").write_text(
|
||||
json.dumps({"features": {"vector_db": True}}), encoding="utf-8"
|
||||
)
|
||||
# Create vectors dir so db_path.exists() is true
|
||||
db_path = get_vector_db_path(vault)
|
||||
db_path.mkdir(parents=True)
|
||||
|
||||
with patch("paperforge.memory.runtime_health.get_embed_status") as m_stat:
|
||||
m_stat.return_value = {
|
||||
"db_exists": True, "chunk_count": 0,
|
||||
"model": "test", "mode": "api",
|
||||
"healthy": False, "corrupted": True, "error": "hnsw error",
|
||||
}
|
||||
with patch("paperforge.memory.runtime_health.get_connection") as m_conn:
|
||||
mock_cursor = m_conn.return_value.execute.return_value
|
||||
mock_cursor.fetchone.side_effect = [(10,), (10,)]
|
||||
result = _check_vector(vault)
|
||||
assert result["status"] == "degraded"
|
||||
assert any("corrupted" in e.lower() for e in result["evidence"])
|
||||
```
|
||||
|
||||
```python
|
||||
def test_check_vector_partial_coverage_returns_degraded(tmp_path):
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir()
|
||||
(vault / "paperforge.json").write_text(
|
||||
json.dumps({"system_dir": "System"}), encoding="utf-8"
|
||||
)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").parent.mkdir(parents=True)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").write_text(
|
||||
json.dumps({"features": {"vector_db": True}}), encoding="utf-8"
|
||||
)
|
||||
from paperforge.embedding import get_vector_db_path
|
||||
db_path = get_vector_db_path(vault)
|
||||
db_path.mkdir(parents=True)
|
||||
|
||||
from paperforge.embedding.build_state import write_vector_build_state
|
||||
write_vector_build_state(vault, {"status": "idle", "current": 0, "total": 0})
|
||||
|
||||
with patch("paperforge.memory.runtime_health.get_embed_status") as m_stat:
|
||||
m_stat.return_value = {
|
||||
"db_exists": True, "chunk_count": 50,
|
||||
"model": "test", "mode": "api",
|
||||
"healthy": True, "corrupted": False, "error": "",
|
||||
}
|
||||
with patch("paperforge.memory.runtime_health.get_connection") as m_conn:
|
||||
mock_cursor = m_conn.return_value.execute.return_value
|
||||
mock_cursor.fetchone.side_effect = [(10,), (7,)]
|
||||
result = _check_vector(vault)
|
||||
assert result["status"] == "degraded"
|
||||
assert any("partial" in e.lower() for e in result["evidence"])
|
||||
```
|
||||
|
||||
```python
|
||||
def test_check_vector_full_coverage_returns_ok(tmp_path):
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir()
|
||||
(vault / "paperforge.json").write_text(
|
||||
json.dumps({"system_dir": "System"}), encoding="utf-8"
|
||||
)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").parent.mkdir(parents=True)
|
||||
(vault / ".obsidian" / "plugins" / "paperforge" / "data.json").write_text(
|
||||
json.dumps({"features": {"vector_db": True}}), encoding="utf-8"
|
||||
)
|
||||
from paperforge.embedding import get_vector_db_path
|
||||
db_path = get_vector_db_path(vault)
|
||||
db_path.mkdir(parents=True)
|
||||
|
||||
from paperforge.embedding.build_state import write_vector_build_state
|
||||
write_vector_build_state(vault, {"status": "idle", "current": 0, "total": 0})
|
||||
|
||||
with patch("paperforge.memory.runtime_health.get_embed_status") as m_stat:
|
||||
m_stat.return_value = {
|
||||
"db_exists": True, "chunk_count": 50,
|
||||
"model": "test", "mode": "api",
|
||||
"healthy": True, "corrupted": False, "error": "",
|
||||
}
|
||||
with patch("paperforge.memory.runtime_health.get_connection") as m_conn:
|
||||
mock_cursor = m_conn.return_value.execute.return_value
|
||||
mock_cursor.fetchone.side_effect = [(10,), (10,)]
|
||||
result = _check_vector(vault)
|
||||
assert result["status"] == "ok"
|
||||
assert "coverage" in result["evidence"][0].lower()
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run tests**
|
||||
Run: `python -m pytest tests/unit/memory/test_runtime_health.py -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 4: Run all unit tests**
|
||||
Run: `python -m pytest tests/unit/ -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(health): strict _check_vector() with coverage ratio"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Auto-embed — Write vector_status to DB after OCR
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/worker/asset_index.py:479-505`
|
||||
|
||||
- [ ] **Step 1: Rewrite `_vec_auto_embed_if_new()` to write status**
|
||||
|
||||
Replace lines 479-505 with:
|
||||
```python
|
||||
def _vec_auto_embed_if_new(vault: Path, entry: dict) -> None:
|
||||
"""Auto-embed a paper into vector DB if OCR is done and vectors missing."""
|
||||
if entry.get("ocr_status") != "done":
|
||||
return
|
||||
fulltext_rel = entry.get("fulltext_path", "")
|
||||
if not fulltext_rel:
|
||||
return
|
||||
fulltext_path = vault / fulltext_rel
|
||||
if not fulltext_path.exists():
|
||||
return
|
||||
try:
|
||||
from paperforge.embedding._config import _read_plugin_settings
|
||||
from paperforge.embedding import embed_paper, get_vector_db_path
|
||||
from paperforge.memory.chunker import chunk_fulltext
|
||||
from paperforge.memory.db import get_connection, get_memory_db_path
|
||||
settings = _read_plugin_settings(vault)
|
||||
if not settings.get("features", {}).get("vector_db", False):
|
||||
return
|
||||
db_path = get_vector_db_path(vault)
|
||||
if not db_path.exists():
|
||||
return
|
||||
chunks = chunk_fulltext(fulltext_path)
|
||||
if not chunks:
|
||||
return
|
||||
|
||||
zotero_key = entry["zotero_key"]
|
||||
embed_paper(vault, zotero_key, chunks)
|
||||
|
||||
# Write vector_status to paperforge.db
|
||||
try:
|
||||
conn = get_connection(get_memory_db_path(vault))
|
||||
conn.execute(
|
||||
"UPDATE papers SET vector_status = 'embedded' WHERE zotero_key = ?",
|
||||
(zotero_key,)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
# Write failure status
|
||||
try:
|
||||
conn = get_connection(get_memory_db_path(vault))
|
||||
conn.execute(
|
||||
"UPDATE papers SET vector_status = 'failed' WHERE zotero_key = ?",
|
||||
(entry["zotero_key"],)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run existing tests**
|
||||
Run: `python -m pytest tests/unit/worker/ -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 3: Run all unit tests**
|
||||
Run: `python -m pytest tests/unit/ -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(auto-embed): write vector_status to DB after OCR"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Agent context — Add vector coverage reporting
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/context.py`
|
||||
- Test: `tests/unit/memory/test_context.py`
|
||||
|
||||
- [ ] **Step 1: Add vector coverage to `get_agent_context()`**
|
||||
|
||||
In `paperforge/memory/context.py`, after the collection tree building (before the `return`), add:
|
||||
|
||||
```python
|
||||
# Vector coverage
|
||||
vector = {}
|
||||
try:
|
||||
total_ocr = conn.execute(
|
||||
"SELECT COUNT(*) FROM papers WHERE ocr_status = 'done'"
|
||||
).fetchone()[0]
|
||||
embedded = conn.execute(
|
||||
"SELECT COUNT(*) FROM papers WHERE ocr_status = 'done' AND vector_status = 'embedded'"
|
||||
).fetchone()[0]
|
||||
# Check if vector DB is enabled in plugin settings
|
||||
vector_enabled = False
|
||||
settings_path = vault / ".obsidian" / "plugins" / "paperforge" / "data.json"
|
||||
if settings_path.exists():
|
||||
import json
|
||||
data = json.loads(settings_path.read_text(encoding="utf-8"))
|
||||
vector_enabled = bool(data.get("features", {}).get("vector_db", False))
|
||||
vector = {
|
||||
"enabled": vector_enabled,
|
||||
"embedded": embedded,
|
||||
"total_ocr_done": total_ocr,
|
||||
"coverage": round(embedded / total_ocr, 4) if total_ocr > 0 else 0.0,
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
```
|
||||
|
||||
Add `"vector": vector` to the returned dict.
|
||||
|
||||
- [ ] **Step 2: Write test**
|
||||
|
||||
Update `tests/unit/memory/test_context.py`:
|
||||
```python
|
||||
def test_get_agent_context_returns_vector_coverage(tmp_path):
|
||||
from paperforge.memory.db import get_connection, get_memory_db_path
|
||||
from paperforge.memory.schema import ensure_schema, CURRENT_SCHEMA_VERSION
|
||||
vault = tmp_path / "vault"
|
||||
vault.mkdir()
|
||||
db_path = get_memory_db_path(vault)
|
||||
conn = get_connection(db_path)
|
||||
ensure_schema(conn)
|
||||
conn.execute("INSERT INTO meta (key, value) VALUES ('schema_version', ?)", (str(CURRENT_SCHEMA_VERSION),))
|
||||
conn.execute("""INSERT INTO papers (zotero_key, title, ocr_status, vector_status)
|
||||
VALUES (?, ?, ?, ?)""",
|
||||
("K1", "Test", "done", "embedded"))
|
||||
conn.execute("""INSERT INTO papers (zotero_key, title, ocr_status, vector_status)
|
||||
VALUES (?, ?, ?, ?)""",
|
||||
("K2", "Test2", "done", ""))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
ctx = get_agent_context(vault)
|
||||
assert ctx is not None
|
||||
assert "vector" in ctx
|
||||
assert ctx["vector"]["embedded"] == 1
|
||||
assert ctx["vector"]["total_ocr_done"] == 2
|
||||
assert ctx["vector"]["coverage"] == 0.5
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run tests**
|
||||
Run: `python -m pytest tests/unit/memory/test_context.py -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
```bash
|
||||
git add -A && git commit -m "feat(context): add vector coverage to agent context"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Full test suite + lint
|
||||
|
||||
- [ ] **Step 1: Run all tests**
|
||||
Run: `python -m pytest tests/unit/ -v --tb=short`
|
||||
Expected: all PASS
|
||||
|
||||
- [ ] **Step 2: Run lint**
|
||||
Run: `ruff check --fix paperforge/ && ruff format paperforge/`
|
||||
Expected: clean
|
||||
|
||||
- [ ] **Step 3: Final commit if lint fixes applied**
|
||||
```bash
|
||||
git add -A && git commit -m "style: lint and format after vector_status changes" # if needed
|
||||
```
|
||||
|
|
@ -0,0 +1,505 @@
|
|||
# PaperForge Stabilization and Complexity-Reduction Design
|
||||
|
||||
**Date:** 2026-05-16
|
||||
**Status:** Proposed
|
||||
**Audience:** Maintainers, contributors, agentic implementers
|
||||
|
||||
---
|
||||
|
||||
## 1. Summary
|
||||
|
||||
PaperForge's architecture direction is sound, but the product currently suffers from four classes of contract drift:
|
||||
|
||||
1. Runtime state and path contracts are not consistently enforced.
|
||||
2. Skill/workflow instructions are not fully aligned with the live CLI surface.
|
||||
3. Documentation mixes audiences and duplicates workflow guidance.
|
||||
4. The normal user experience exposes too much implementation complexity too early.
|
||||
|
||||
This design proposes a medium-scope stabilization program that prioritizes existing power users first. The program explicitly does **not** begin with a frontend rewrite. Instead, it hardens runtime contracts, aligns the command and skill surfaces, rebuilds documentation information architecture, and only then reduces user-facing complexity through progressive disclosure.
|
||||
|
||||
---
|
||||
|
||||
## 2. Product Decision
|
||||
|
||||
### Chosen Priority
|
||||
|
||||
- **Primary audience for this program:** existing power users
|
||||
- **Refactor intensity:** medium
|
||||
- **Principle:** do not rewrite the frontend first
|
||||
|
||||
### Why
|
||||
|
||||
The most dangerous current failures are not discoverability issues alone, but cases where the system appears healthy while state, commands, or paths silently drift. Improving onboarding before stabilizing these contracts would increase user trust in workflows that are not yet reliable enough.
|
||||
|
||||
---
|
||||
|
||||
## 3. Problem Statement
|
||||
|
||||
PaperForge is currently "strong but brittle":
|
||||
|
||||
- Runtime snapshots are treated as a contract by the plugin, but not fully implemented as contract-grade files.
|
||||
- Index mutation paths are not fully serialized, enabling stale or overwritten state.
|
||||
- `sync` can publish pre-clean rather than post-clean truth.
|
||||
- The plugin still hardcodes `System/PaperForge/...` in key places, breaking the real meaning of configurable paths.
|
||||
- Agent skill workflows contain stale commands, stale enums, stale paths, and ambiguous router behavior.
|
||||
- `AGENTS.md`, `README*`, and `docs/COMMANDS.md` collide in purpose and repeat the same flows.
|
||||
- New users are exposed to too many layers at once: runtime, OCR, Zotero storage linking, memory layer, vector DB, agent setup, dashboard modes, frontmatter toggles, and chat commands.
|
||||
|
||||
The result is a system with high capability but avoidable operational risk and cognitive overhead.
|
||||
|
||||
---
|
||||
|
||||
## 4. Goals
|
||||
|
||||
### 4.1 Primary Goals
|
||||
|
||||
1. Eliminate silent state drift across plugin, index, DB, and filesystem.
|
||||
2. Make the CLI the single truth source for all documented and skill-invoked commands.
|
||||
3. Separate user, agent, and maintainer documentation cleanly.
|
||||
4. Reduce complexity of the first-value path without large UI surgery.
|
||||
|
||||
### 4.2 Secondary Goals
|
||||
|
||||
1. Improve confidence in `/pf-*` routing and workflow reliability.
|
||||
2. Make custom path configuration genuinely supported end-to-end.
|
||||
3. Reduce future maintenance cost by clarifying truth boundaries.
|
||||
|
||||
### 4.3 Non-Goals
|
||||
|
||||
1. No full dashboard redesign.
|
||||
2. No broad `main.js` extraction project unless needed to support a chosen package.
|
||||
3. No major new features.
|
||||
4. No attempt to make PaperForge beginner-perfect in one pass.
|
||||
|
||||
---
|
||||
|
||||
## 5. Guiding Principles
|
||||
|
||||
1. **CLI is the command truth source.** Skill docs, user docs, plugin guidance, and examples must all match the live CLI surface.
|
||||
2. **Python is the runtime truth source.** JS reads canonical runtime state; it does not infer it.
|
||||
3. **One document, one audience.** README, tutorial, AGENTS, command reference, and maintainer docs must not overlap in purpose.
|
||||
4. **First value before full configuration.** Users should be able to reach first sync before OCR, vector DB, or deep agent setup becomes mandatory.
|
||||
5. **Mechanical commands and thinking workflows are different classes.** `/pf-sync`, `/pf-ocr`, `/pf-status` must not be treated like `/pf-deep` and `/pf-paper`.
|
||||
|
||||
---
|
||||
|
||||
## 6. Proposed Program Structure
|
||||
|
||||
The work is split into four packages executed in order.
|
||||
|
||||
### 6.0 Package Ownership Rules
|
||||
|
||||
| Package | Allowed changes | Disallowed changes | Owned files | Cross-package rule |
|
||||
|---|---|---|---|---|
|
||||
| A | runtime state writes, path resolution, sync truth, lifecycle control, regression tests | UX copy cleanup, documentation restructuring, information architecture changes | runtime Python modules, plugin runtime path/state logic | May touch `paperforge/plugin/main.js` only for runtime path/state behavior |
|
||||
| B | CLI/skill/workflow alignment, route ownership, command examples, validation tests | runtime contract redesign, doc IA restructuring, UX layout changes | `paperforge/cli.py`, relevant commands, `paperforge/skills/paperforge/*` | May depend on A outputs, but does not redefine A runtime behavior |
|
||||
| C | README shrink, tutorial/troubleshooting split, AGENTS narrowing, maintainer doc ownership | runtime behavior changes, route logic changes, UI behavior changes | README, AGENTS, docs tree | Owns documentation structure and glossary decisions |
|
||||
| D | staged setup gating, visibility toggles, primary action placement, user-facing error copy, terminology adoption in UI | runtime contract changes, command-surface changes, full visual redesign, component extraction project | plugin presentation behavior and user-facing copy | Must consume frozen outputs from C and cannot restructure docs |
|
||||
|
||||
#### Boundary Invariants
|
||||
|
||||
- Package A may touch plugin code only for runtime path/state correctness.
|
||||
- Package D may touch plugin code only for presentation, visibility, and copy behavior.
|
||||
- `README*` ownership belongs to Package C only.
|
||||
- `AGENTS.md` ownership belongs to Package C only.
|
||||
- Package D may consume terminology decisions from Package C, but may not redefine them.
|
||||
|
||||
### Package A: Runtime Contract Hardening
|
||||
|
||||
**Goal:** make runtime state, path resolution, and end-of-command truth reliable.
|
||||
|
||||
#### Scope
|
||||
|
||||
- Convert all runtime snapshot writes to atomic writes.
|
||||
- Create one safe mutation path for `formal-library.json` updates.
|
||||
- Define `sync` final state as cleaned, canonical end state.
|
||||
- Remove plugin hardcoded assumptions about `System/PaperForge/...`.
|
||||
- Fix runtime lifecycle gaps such as `embed stop`, snapshot bootstrap semantics, and startup update behavior.
|
||||
|
||||
#### Required Outcomes
|
||||
|
||||
- Plugin never reads half-written snapshot JSON.
|
||||
- Path overrides from `paperforge.json` are respected end-to-end.
|
||||
- `sync` no longer leaves pre-clean truth published as final truth.
|
||||
- `embed stop` returns success only when it has either issued a real stop signal or explicitly reports that no stoppable job exists.
|
||||
- First-launch bootstrap generates every runtime snapshot file the plugin checks during startup.
|
||||
|
||||
#### Files in Scope
|
||||
|
||||
- `paperforge/memory/state_snapshot.py`
|
||||
- `paperforge/memory/vector_db.py`
|
||||
- `paperforge/core/io.py`
|
||||
- `paperforge/worker/asset_index.py`
|
||||
- `paperforge/worker/repair.py`
|
||||
- `paperforge/services/sync_service.py`
|
||||
- `paperforge/memory/refresh.py`
|
||||
- `paperforge/commands/embed.py`
|
||||
- `paperforge/commands/runtime_health.py`
|
||||
- `paperforge/plugin/main.js`
|
||||
|
||||
### Package B: Command & Skill Truth Alignment
|
||||
|
||||
**Goal:** eliminate stale or incorrect command instructions and router ambiguity.
|
||||
|
||||
#### Scope
|
||||
|
||||
- Audit live CLI surface in `paperforge/cli.py` and relevant command modules.
|
||||
- Redefine compound skill routing so mechanical commands have explicit handling.
|
||||
- Correct stale workflow examples, enum values, render paths, and fallback logic.
|
||||
- Ensure `agent-context` exposed usage strings match the real command surface.
|
||||
|
||||
#### Required Outcomes
|
||||
|
||||
- Every documented workflow command can actually run.
|
||||
- Every `/pf-*` route has one clear owner.
|
||||
- Workflow docs no longer reference nonexistent flags or stale output paths.
|
||||
- CI or fixture validation detects if workflow docs or agent-context examples reference commands or flags not present in the live CLI surface.
|
||||
|
||||
#### Files in Scope
|
||||
|
||||
- `paperforge/cli.py`
|
||||
- `paperforge/commands/paper_context.py`
|
||||
- `paperforge/commands/paper_status.py`
|
||||
- `paperforge/commands/search.py`
|
||||
- `paperforge/commands/reading_log.py`
|
||||
- `paperforge/commands/project_log.py`
|
||||
- `paperforge/commands/agent_context.py`
|
||||
- `paperforge/memory/runtime_health.py`
|
||||
- `paperforge/skills/paperforge/SKILL.md`
|
||||
- `paperforge/skills/paperforge/workflows/*.md`
|
||||
|
||||
### Package C: Documentation IA Reset
|
||||
|
||||
**Goal:** separate audiences and establish one authoritative workflow tutorial.
|
||||
|
||||
#### Scope
|
||||
|
||||
- Shrink `README.md` and `README.en.md` into entry pages.
|
||||
- Create a canonical end-user tutorial.
|
||||
- Create a dedicated troubleshooting document.
|
||||
- Convert `AGENTS.md` into agent-only operating guidance.
|
||||
- Convert `docs/COMMANDS.md` into pure command reference.
|
||||
- Move maintainer architecture, release, testing, and versioning guidance into maintainer-facing docs.
|
||||
|
||||
#### Required Outcomes
|
||||
|
||||
- Users do not need `AGENTS.md` to use the product.
|
||||
- Agents do not rely on user tutorial content for repository operating rules.
|
||||
- One workflow is documented once, then referenced elsewhere.
|
||||
- Stale links, stale version markers, and audience collisions are removed.
|
||||
- Ownership is explicit:
|
||||
- `docs/getting-started.md` owns the end-user happy path
|
||||
- `docs/troubleshooting.md` owns user failure recovery
|
||||
- `docs/COMMANDS.md` owns CLI reference only
|
||||
- `docs/maintainer-guide.md` owns release, testing, versioning, migration, and architecture links
|
||||
|
||||
#### Files in Scope
|
||||
|
||||
- `README.md`
|
||||
- `README.en.md`
|
||||
- `AGENTS.md`
|
||||
- `docs/COMMANDS.md`
|
||||
- `docs/ARCHITECTURE.md`
|
||||
- new `docs/getting-started.md`
|
||||
- new `docs/troubleshooting.md`
|
||||
- new `docs/maintainer-guide.md`
|
||||
|
||||
### Package D: Progressive Disclosure UX
|
||||
|
||||
**Goal:** reduce user complexity without a frontend rewrite.
|
||||
|
||||
#### Scope
|
||||
|
||||
- Make the setup wizard staged instead of fully front-loaded.
|
||||
- Fix global/home surface so core mechanical actions are consistently visible.
|
||||
- Hide advanced memory/vector/runtime surfaces until needed.
|
||||
- Rewrite user-facing error messages to lead with action.
|
||||
- Normalize core product terms across UI and docs.
|
||||
|
||||
#### Required Outcomes
|
||||
|
||||
- New users can reach first sync before confronting OCR/agent/vector configuration.
|
||||
- The Home/Global surface exposes `Sync`, `OCR`, `Status`, `Doctor`, and `Repair` in one persistent primary action group on first run and after setup completion.
|
||||
- Advanced memory/vector/runtime controls are hidden behind one collapsed `Advanced` section by default.
|
||||
- A controlled glossary of core product terms is defined in Package C, and user-visible strings in plugin UI adopt those exact terms.
|
||||
|
||||
#### Files in Scope
|
||||
|
||||
- `paperforge/plugin/main.js`
|
||||
|
||||
#### In Scope
|
||||
|
||||
- staged setup gating
|
||||
- action placement
|
||||
- visibility toggles
|
||||
- user-facing copy
|
||||
- terminology normalization
|
||||
|
||||
#### Out of Scope
|
||||
|
||||
- new navigation architecture
|
||||
- component extraction project
|
||||
- visual redesign
|
||||
- settings subsystem rewrite
|
||||
- dashboard information model redesign
|
||||
|
||||
---
|
||||
|
||||
## 7. Execution Order
|
||||
|
||||
### Required Order
|
||||
|
||||
1. Package A
|
||||
2. Package B
|
||||
3. Package C
|
||||
4. Package D
|
||||
|
||||
### Rationale
|
||||
|
||||
- Package A stabilizes truth.
|
||||
- Package B stabilizes command semantics built on that truth.
|
||||
- Package C stabilizes explanations of those semantics.
|
||||
- Package D improves surface complexity only after the lower layers stop moving.
|
||||
|
||||
### Release Strategy
|
||||
|
||||
- **Milestone 1:** Package A only, released as a stability hardening version.
|
||||
- **Milestone 2:** Package B only, released as command and agent alignment.
|
||||
- **Milestone 3:** Package C only, released as documentation IA reset.
|
||||
- **Milestone 4:** Package D only, released as progressive-disclosure UX simplification.
|
||||
|
||||
This sequencing keeps release narratives clean and reduces regression blast radius.
|
||||
|
||||
---
|
||||
|
||||
## 8. Design Details by Risk Area
|
||||
|
||||
### 8.1 Snapshot Contract
|
||||
|
||||
Snapshots are now part of the runtime contract and must be treated as such.
|
||||
|
||||
#### Decision
|
||||
|
||||
- All snapshot writes become atomic (`.tmp` then replace).
|
||||
- Snapshot readers may still degrade gracefully, but parse failure should become exceptional rather than expected.
|
||||
- The first-launch bootstrap must generate the files the plugin is actually checking for.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- No direct `write_text(json.dumps(...))` remains for contract snapshots.
|
||||
- Plugin startup no longer relies on a bootstrap command that writes a different file than the one being checked.
|
||||
|
||||
### 8.2 Canonical Index Mutation
|
||||
|
||||
`formal-library.json` must not be mutated through multiple partially safe paths.
|
||||
|
||||
#### Decision
|
||||
|
||||
- Introduce one canonical mutation path for read-modify-write behavior.
|
||||
- The safety boundary includes read, merge, write, and replace, not only the last write.
|
||||
- Repair and incremental refresh should use the same safety primitive.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- No code path directly writes the canonical index outside the shared mutation layer.
|
||||
- Concurrency-sensitive tests exist for overlapping update scenarios.
|
||||
|
||||
### 8.3 `sync` Final-State Truth
|
||||
|
||||
The system must choose whether sync publishes pre-clean or post-clean truth. This design chooses **post-clean truth only**.
|
||||
|
||||
#### Decision
|
||||
|
||||
- Cleanup is part of the sync transaction semantics.
|
||||
- `formal-library.json` and DB refreshes must reflect the cleaned end state, not an intermediate build stage.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- A completed sync cannot leave deleted notes present in the canonical index or memory DB.
|
||||
- Sync tests assert cleaned-end-state semantics explicitly.
|
||||
|
||||
### 8.4 Path Configuration Contract
|
||||
|
||||
Custom path configuration is either real or it is not. This design treats it as real.
|
||||
|
||||
#### Decision
|
||||
|
||||
- The plugin must resolve runtime files through one configuration-aware path source.
|
||||
- Hardcoded `System/PaperForge` assumptions in runtime reads, polling, and bootstrap logic are removed.
|
||||
|
||||
#### Authority and Bootstrap Contract
|
||||
|
||||
- **Authority:** resolved runtime paths derive from vault root + `paperforge.json` configuration, using one configuration-aware resolution flow.
|
||||
- **Bootstrap:** plugin startup must load config before constructing runtime file paths.
|
||||
- **Fallback:** if config is missing or corrupt, plugin falls back to documented defaults and surfaces a recoverable warning instead of silently constructing mixed paths.
|
||||
- **Invariant:** plugin runtime path construction must not use hardcoded `System/PaperForge/...` literals outside the shared path resolution flow.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- A vault with non-default `system_dir` still gets working snapshots, export polling, OCR polling, and dashboard stats.
|
||||
|
||||
### 8.5 Mechanical vs Cognitive Routes
|
||||
|
||||
The skill layer must reflect the actual distinction in the product.
|
||||
|
||||
#### Decision
|
||||
|
||||
- `/pf-sync`, `/pf-ocr`, `/pf-status` are mechanical execution routes.
|
||||
- `/pf-deep`, `/pf-paper` are cognitive workflows.
|
||||
- The compound skill router must represent this explicitly.
|
||||
|
||||
#### Routing Ownership Matrix
|
||||
|
||||
| Route | Owner | Behavior |
|
||||
|---|---|---|
|
||||
| `/pf-sync` | mechanical executor | run CLI sync path and return execution/result interpretation |
|
||||
| `/pf-ocr` | mechanical executor | run CLI OCR path and return execution/result interpretation |
|
||||
| `/pf-status` | mechanical executor | run CLI status/health path and return execution/result interpretation |
|
||||
| `/pf-deep` | cognitive workflow | run deep-reading workflow with runtime gating |
|
||||
| `/pf-paper` | cognitive workflow | run paper-qa workflow with identifier resolution first |
|
||||
|
||||
Unknown or malformed slash commands must not fall through silently into `project-engineering`; they should trigger explicit clarification or an unsupported-command response.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- No slash command relies on accidental fallback into `project-engineering`.
|
||||
- Known slash commands map to exactly one owner.
|
||||
- Unknown slash commands produce a deterministic clarification path.
|
||||
- Workflow routing examples match live behavior.
|
||||
|
||||
### 8.6 Documentation Audience Separation
|
||||
|
||||
Current documentation is content-rich but structurally unsound.
|
||||
|
||||
#### Decision
|
||||
|
||||
- `AGENTS.md` becomes repository operating guidance only.
|
||||
- Tutorial ownership moves to a dedicated getting-started guide.
|
||||
- README becomes a navigation page, not the entire product manual.
|
||||
- `docs/maintainer-guide.md` is required, not optional.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- A normal user can complete the main workflow without opening `AGENTS.md`.
|
||||
- A contributor/agent can learn repo operating rules without reading end-user tutorial content.
|
||||
- Installation, upgrade, release compatibility, and migration ownership are assigned to maintainer-facing docs rather than spread across README and AGENTS.
|
||||
|
||||
### 8.7 UX Simplification Without Rewrite
|
||||
|
||||
This program reduces complexity via staging and hiding, not by redesigning every screen.
|
||||
|
||||
#### Decision
|
||||
|
||||
- The first-value path ends at sync.
|
||||
- OCR, agent setup, memory layer, and vector DB are later-stage surfaces.
|
||||
- Global/Home view must always expose core mechanical actions.
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
- The setup flow no longer blocks first sync on OCR or advanced features.
|
||||
- On first visit after setup, runtime/setup/core workflow remain visible while advanced memory/vector/runtime controls are collapsed by default.
|
||||
- OCR, agent, memory, and vector configuration are not required to complete first sync.
|
||||
|
||||
---
|
||||
|
||||
## 9. Compatibility and Migration Considerations
|
||||
|
||||
### 9.1 Existing Users
|
||||
|
||||
- Existing power users should not lose access to advanced runtime, memory, or vector controls.
|
||||
- Advanced controls may move behind clearer grouping, but not disappear.
|
||||
|
||||
### 9.2 Existing Documentation Links
|
||||
|
||||
- README and AGENTS restructuring will require link updates across docs.
|
||||
- Redirect-by-reference is sufficient; no file-system redirect mechanism is needed.
|
||||
|
||||
### 9.3 Existing Agent Habits
|
||||
|
||||
- Some users may already treat `/pf-sync` as a broad engineering discussion trigger.
|
||||
- After alignment, mechanical slash commands should act more literally and predictably.
|
||||
|
||||
### 9.4 Path Overrides
|
||||
|
||||
- Supporting custom `system_dir` fully means tests and plugin reads must cover non-default path layouts, not only defaults.
|
||||
|
||||
---
|
||||
|
||||
## 10. Risks
|
||||
|
||||
1. **Over-expansion risk:** this program can accidentally become a full frontend refactor unless file boundaries are enforced.
|
||||
2. **Contract churn risk:** changing plugin path logic and sync semantics simultaneously can create regression clusters if not tested incrementally.
|
||||
3. **Documentation drift risk during transition:** if docs are edited before command alignment is done, they will drift again.
|
||||
4. **Behavioral surprise risk for power users:** moving advanced controls or changing slash-command routing may break habit memory if changelog messaging is weak.
|
||||
|
||||
### Mitigations
|
||||
|
||||
- Ship in four milestones.
|
||||
- Treat Package A and Package B as contract work with explicit regression tests.
|
||||
- Do not start Package D until Package C terminology is settled.
|
||||
- Use changelog notes to explain route and doc entry changes.
|
||||
|
||||
---
|
||||
|
||||
## 11. Testing Strategy
|
||||
|
||||
### Package A
|
||||
|
||||
- Unit tests for atomic snapshot write helpers.
|
||||
- Integration tests for concurrent index mutation paths.
|
||||
- Integration tests for sync post-clean final-state semantics.
|
||||
- Plugin-facing tests for non-default path resolution assumptions where feasible.
|
||||
|
||||
### Package B
|
||||
|
||||
- Command-surface verification tests.
|
||||
- Workflow reference audit tests or fixture-based validation where practical.
|
||||
- Manual verification of each `/pf-*` route against documented behavior.
|
||||
- CI validation that skill/workflow examples and `agent-context` command references do not mention nonexistent commands or flags.
|
||||
|
||||
### Package C
|
||||
|
||||
- Link integrity pass.
|
||||
- Manual audience review: user flow, agent flow, maintainer flow.
|
||||
|
||||
### Package D
|
||||
|
||||
- Manual first-run walkthrough.
|
||||
- Manual regression on advanced settings visibility.
|
||||
- String/term consistency pass across docs and UI.
|
||||
|
||||
---
|
||||
|
||||
## 12. Success Criteria
|
||||
|
||||
This program is successful if all of the following are true:
|
||||
|
||||
1. The plugin and Python package agree on path layout and runtime state without hardcoded default assumptions.
|
||||
2. `sync`, `repair`, and incremental updates no longer leave contradictory truth between filesystem, index, and DB.
|
||||
3. Every shipped skill/workflow command reference matches the live CLI.
|
||||
4. `AGENTS.md` is no longer required reading for end users.
|
||||
5. There is exactly one canonical user tutorial.
|
||||
6. A user can reach first sync without being forced through OCR, vector DB, or deep agent setup.
|
||||
7. Documentation ownership is explicit and implemented by file structure, not only by prose intent.
|
||||
|
||||
---
|
||||
|
||||
## 13. Out of Scope Follow-Ups
|
||||
|
||||
These are intentionally deferred until after this program:
|
||||
|
||||
- Large-scale `main.js` decomposition
|
||||
- full Settings/UI component extraction
|
||||
- broader design polish work
|
||||
- new workflows or major memory/vector features
|
||||
|
||||
---
|
||||
|
||||
## 14. Implementation Handoff
|
||||
|
||||
The implementation plan should preserve the four-package structure and convert it into dependency-aware tasks with:
|
||||
|
||||
- explicit file lists
|
||||
- regression-first testing
|
||||
- milestone-based execution
|
||||
- no large opportunistic refactors outside scope
|
||||
209
docs/superpowers/specs/2026-05-18-discussion-md-truth-source.md
Normal file
209
docs/superpowers/specs/2026-05-18-discussion-md-truth-source.md
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
# Discussion MD as Truth Source — Design Spec
|
||||
|
||||
> **Status:** Draft | **Date:** 2026-05-18
|
||||
> **Motivation:** `discussion.json` forces rich markdown content (tables, code blocks, callouts, ASCII flowcharts) into JSON strings, destroying formatting and making output unreadable. Switch to `discussion.md` as the sole truth source.
|
||||
|
||||
## Goal
|
||||
|
||||
1. **Stop writing `discussion.json`** — QA content lives only in `.md`
|
||||
2. **Keep `discussion.md` as the rich markdown output** — no `_escape_md()` on answer text
|
||||
3. **Plugin card reads `.md` directly** — parse the last 3 Q&A pairs from markdown structure
|
||||
4. **Existing `.json` files are ignored** — no migration needed
|
||||
|
||||
---
|
||||
|
||||
## Current Architecture (as-is)
|
||||
|
||||
```
|
||||
Agent workflow (paper-qa.md)
|
||||
↓ --qa-pairs '<JSON string>'
|
||||
discussion.py record_session()
|
||||
↓
|
||||
├── ai/discussion.json ← canonical (structured, sessions[], qa_pairs[])
|
||||
├── ai/discussion.md ← human-readable (escaped, no rich formatting)
|
||||
│
|
||||
plugin main.js
|
||||
├── _renderRecentDiscussionCard() ← reads discussion.json
|
||||
└── "查看全部讨论" link ← opens discussion.md
|
||||
```
|
||||
|
||||
**Problem:** `_escape_md()` escapes `**bold**` → `\*\*bold\*\*`, tables, code blocks, callouts all get destroyed. The `.md` file is barely readable.
|
||||
|
||||
---
|
||||
|
||||
## Target Architecture (to-be)
|
||||
|
||||
```
|
||||
Agent workflow (paper-qa.md)
|
||||
↓ --qa-pairs '<JSON string>' (same API, no change)
|
||||
discussion.py record_session()
|
||||
↓
|
||||
└── ai/discussion.md ← truth source (RICH markdown, no escaping)
|
||||
│
|
||||
plugin main.js
|
||||
├── _renderRecentDiscussionCard() ← reads discussion.md, parses last 3 Q&A
|
||||
└── "查看全部讨论" link ← opens discussion.md (same)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## File Changes
|
||||
|
||||
### Part A: `paperforge/worker/discussion.py`
|
||||
|
||||
**Schema:**
|
||||
- Remove `_escape_md()` — answer text goes directly into `.md` unescaped
|
||||
- Remove `_atomic_write_json()` — no more `.json` writing
|
||||
- `_build_md_session()` — no longer calls `_escape_md()` on answer text
|
||||
- `record_session()` — remove JSON reading/writing, only lock + write `.md`
|
||||
- Return value: remove `json_path`, keep `md_path`
|
||||
- Lock file: change from `discussion.json.lock` to `discussion.md.lock`
|
||||
|
||||
**New `record_session()` flow:**
|
||||
```
|
||||
1. Validate inputs (same)
|
||||
2. Find paper metadata (same)
|
||||
3. Build paths: ai_dir + /discussion.md (no json_path)
|
||||
4. Build session dict (same, but only used for md generation)
|
||||
5. Acquire lock on discussion.md.lock
|
||||
6. Read existing discussion.md (if exists)
|
||||
7. Append new session markdown
|
||||
8. Atomic write discussion.md
|
||||
9. Release lock
|
||||
10. Return {"status": "ok", "md_path": str(md_path)}
|
||||
```
|
||||
|
||||
### Part B: `paperforge/plugin/main.js`
|
||||
|
||||
**`_renderRecentDiscussionCard()` — new logic:**
|
||||
|
||||
Instead of reading `discussion.json`, read `discussion.md` and render with `MarkdownRenderer.render()`:
|
||||
|
||||
1. Read `wsDir + '/ai/discussion.md'`
|
||||
2. Split content by `## ` headings (session separator)
|
||||
3. Take the last session section
|
||||
4. Within that section, split by `**问题:**` to extract QA pairs
|
||||
5. Take the last 3 QA pairs
|
||||
6. For each QA pair:
|
||||
- Render question as `**提问:** ...` via `MarkdownRenderer.render()`
|
||||
- Render answer as formatted markdown via `MarkdownRenderer.render()`
|
||||
- No manual truncation — API renders native Obsidian markdown (tables, code blocks, callouts all work)
|
||||
7. For long answers (>500 chars): wrap in `max-height: 200px` + `overflow: hidden` with "展开更多" toggle
|
||||
8. "查看全部讨论" link unchanged
|
||||
|
||||
**QA pair parser (extract from .md):**
|
||||
|
||||
```js
|
||||
function _parseDiscussionMD(content) {
|
||||
const sessions = content.split(/\n## /).slice(1);
|
||||
if (sessions.length === 0) return null;
|
||||
const lastSession = sessions[sessions.length - 1];
|
||||
const pairs = [];
|
||||
const qaBlocks = lastSession.split(/\*\*问题:\*\*/).slice(1);
|
||||
for (const block of qaBlocks) {
|
||||
const answerMatch = block.match(/\*\*解答:\*\*/);
|
||||
if (!answerMatch) continue;
|
||||
const question = block.substring(0, answerMatch.index).trim();
|
||||
const answer = block.substring(answerMatch.index + '**解答:**'.length).trim();
|
||||
pairs.push({ question, answer });
|
||||
}
|
||||
return pairs.slice(-3);
|
||||
}
|
||||
```
|
||||
|
||||
**Render each QA pair with MarkdownRenderer:**
|
||||
|
||||
```js
|
||||
async function _renderQAPair(container, qa, sourcePath) {
|
||||
// Question
|
||||
const qEl = container.createEl('div', { cls: 'paperforge-discussion-q' });
|
||||
await MarkdownRenderer.render(
|
||||
this.app,
|
||||
'**提问：**' + qa.question,
|
||||
qEl, sourcePath, this
|
||||
);
|
||||
|
||||
// Answer
|
||||
const aEl = container.createEl('div', { cls: 'paperforge-discussion-a' });
|
||||
if (qa.answer.length > 500) {
|
||||
aEl.style.maxHeight = '200px';
|
||||
aEl.style.overflow = 'hidden';
|
||||
// Add "展开更多" toggle
|
||||
const toggle = container.createEl('button', { text: '展开更多 ▽' });
|
||||
toggle.addEventListener('click', () => {
|
||||
aEl.style.maxHeight = aEl.style.maxHeight ? '' : '200px';
|
||||
toggle.setText(aEl.style.maxHeight ? '展开更多 ▽' : '收起 △');
|
||||
});
|
||||
}
|
||||
await MarkdownRenderer.render(
|
||||
this.app,
|
||||
qa.answer,
|
||||
aEl, sourcePath, this
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
**Lifecycle management:**
|
||||
- The `this` context in `_renderRecentDiscussionCard()` is the plugin instance (extends `Plugin` which extends `Component`)
|
||||
- Pass `this` as the `component` parameter — ensures proper cleanup when card content changes
|
||||
- No need to create a separate `Component` instance
|
||||
|
||||
### Part C: `paperforge/skills/paperforge/workflows/paper-qa.md`
|
||||
|
||||
- Update line 8: `discussion.json` → `discussion.md`
|
||||
- No other changes — the CLI command `python -m paperforge.worker.discussion record` keeps same interface
|
||||
|
||||
### Part D: Tests
|
||||
|
||||
`tests/test_discussion.py`:
|
||||
- `test_create_both_files` → `test_create_md_file` (remove JSON assertions, verify .md has rich content)
|
||||
- `test_append_second_session` → verify .md appends correctly
|
||||
- `test_markdown_escaping` → **REVERSE** this test: verify that `**bold**` is preserved (not escaped)
|
||||
- `test_markdown_escaping_cjk` → same: verify CJK + unescaped markdown coexists
|
||||
- `test_file_lock` → use `.md.lock` instead of `.json.lock`
|
||||
- `test_cli_invocation` → no longer checks json_path
|
||||
- Remove `test_utc_timestamp` (no JSON to check)
|
||||
|
||||
### Part E: Backward Compatibility
|
||||
|
||||
- Old `discussion.json` files are **left in place**, not deleted
|
||||
- Old `discussion.md` files (with escaped content) will be parsed by the new card — the Q&A text will have `\*` characters, but this is acceptable
|
||||
- Over time, as new sessions are appended, the card shows the latest (unescaped) content
|
||||
- No migration needed
|
||||
|
||||
---
|
||||
|
||||
## Data Flow
|
||||
|
||||
### Write path (unchanged except no JSON)
|
||||
|
||||
```
|
||||
Agent: --qa-pairs '[
|
||||
{"question": "What is X?",
|
||||
"answer": "X is **significant** (p<0.05)\n\n| Group | Mean |\n|------|------|\n| A | 10 |"} ]'
|
||||
|
||||
discussion.py record_session():
|
||||
→ Build md session with raw markdown (no escaping)
|
||||
→ Atomic append to discussion.md
|
||||
→ Return {md_path: "..."}
|
||||
```
|
||||
|
||||
### Read path (new)
|
||||
|
||||
```
|
||||
plugin main.js:
|
||||
→ this.app.vault.adapter.read('ai/discussion.md')
|
||||
→ split by "## " → take last session
|
||||
→ split by "**问题:**" → take last 3 QA pairs
|
||||
→ MarkdownRenderer.render(app, qa.answer, el, sourcePath, this)
|
||||
→ Native Obsidian rendering: tables, code blocks, callouts, bold all visible
|
||||
→ max-height: 200px for long answers (>500 chars) with "展开更多"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- Deleting old `discussion.json` files
|
||||
- Migrating existing escaped content to unescaped
|
||||
- Adding a "copy as JSON" button or API
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
# Memory Layer Simplification — Design Spec
|
||||
|
||||
> **Status:** Draft | **Date:** 2026-05-18
|
||||
> **Review:** Pending
|
||||
|
||||
## Goal
|
||||
|
||||
Simplify the memory/embedding layer by: (1) extracting vector DB logic from `memory/vector_db.py` into a new `embedding/` package, (2) removing local ChromaDB + sentence-transformers embedding model support entirely, keeping only API-based embedding, (3) ensuring zero breakage across all consumers.
|
||||
|
||||
## Motivation
|
||||
|
||||
The current `memory/vector_db.py` (364 lines) mixes ChromaDB local mode, sentence-transformers model loading, OpenAI API embedding, build state management, and retrieval — all in one file. This creates unnecessary dependency surface (`chromadb`, `sentence_transformers`, `torch`, `transformers`) and architectural coupling.
|
||||
|
||||
PaperForge is a **literature workflow engine**, not a local vector database server. API embedding via OpenAI-compatible providers is sufficient for semantic search enhancement. Local mode adds ~2GB of dependencies and significant maintenance burden with no proportional benefit.
|
||||
|
||||
## Constraint: Plugin is Immutable
|
||||
|
||||
The Obsidian plugin (`plugin/main.js`) reads exactly these JSON snapshot files:
|
||||
- `System/PaperForge/indexes/memory-runtime-state.json`
|
||||
- `System/PaperForge/indexes/vector-runtime-state.json`
|
||||
- `System/PaperForge/indexes/runtime-health.json`
|
||||
- `System/PaperForge/indexes/vector-build-state.json`
|
||||
|
||||
These files **must continue to exist at their current paths with the same schema**. No plugin code changes are allowed. The refactoring is Python-only.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Before:
|
||||
memory/vector_db.py (364 lines — ChromaDB + ST + OpenAI + build state + retrieve + status)
|
||||
worker/vector_db.py (92 lines — preflight + embed status)
|
||||
worker/asset_index.py:442 (inline auto-embed during sync)
|
||||
|
||||
After:
|
||||
embedding/ (NEW package — API-only embedding)
|
||||
__init__.py — public exports for all consumers
|
||||
_config.py — settings reader (.env + plugin data.json)
|
||||
providers/
|
||||
__init__.py
|
||||
base.py — EmbeddingProvider ABC
|
||||
openai_compatible.py — OpenAI-compatible API client
|
||||
builder.py — embed_paper() via API only
|
||||
search.py — retrieve_chunks() via API only
|
||||
build_state.py — build state persistence (read/write/mark)
|
||||
status.py — get_embed_status()
|
||||
preflight.py — _preflight_check()
|
||||
|
||||
memory/vector_db.py → deprecated shim (forwards to embedding/)
|
||||
worker/vector_db.py → deprecated shim (forwards to embedding/)
|
||||
|
||||
memory/state_snapshot.py — UNCHANGED (still writes 3 legacy snapshot files)
|
||||
memory/ — UNCHANGED: db.py, schema.py, builder.py, fts.py,
|
||||
query.py, context.py, chunker.py, permanent.py,
|
||||
events.py, refresh.py, runtime_health.py, _columns.py
|
||||
```
|
||||
|
||||
## Key Decisions
|
||||
|
||||
### D1: Delete local embedding mode entirely
|
||||
|
||||
**Decision:** Remove ChromaDB for local embedding calculation (sentence-transformers, `BAAI/bge-small-en-v1.5` model, `_get_st()`, `get_embedding_model()`, `_download_model_via_mirror()`). Keep ChromaDB **only as a vector store** — embeddings are generated exclusively via API.
|
||||
|
||||
**Rationale:**
|
||||
- ChromaDB as a vector store is fine (~30MB)
|
||||
- sentence-transformers is the problem (~500MB + torch)
|
||||
- Users who want local embedding can point to a local OpenAI-compatible endpoint (e.g., llamafile, Ollama, vLLM)
|
||||
- `openai` + `chromadb` are the only remaining deps
|
||||
|
||||
### D2: ChromaDB stays as vector store
|
||||
|
||||
**Decision:** Retain ChromaDB `PersistentClient` + `get_collection()` for on-disk vector storage. Only the embedding *source* changes from local model to API.
|
||||
|
||||
**Impact:** `_get_chroma()` is kept (moved to `_chroma.py`). `_get_st()`, `get_embedding_model()`, `_download_model_via_mirror()` are deleted.
|
||||
|
||||
### D3: Snapshot files stay at their current schema
|
||||
|
||||
**Decision:** `write_memory_runtime()`, `write_vector_runtime()`, `write_runtime_health()` keep their exact signatures. `state_snapshot.py` is not modified. This guarantees zero plugin breakage.
|
||||
|
||||
**Impact:** The old plan's "unified runtime-state.json" is removed. The 3 legacy files continue to be written from exactly the same call sites.
|
||||
|
||||
### D4: Deprecated shims for backward compatibility
|
||||
|
||||
**Decision:** `memory/vector_db.py` and `worker/vector_db.py` become forwarding shims that emit `DeprecationWarning` and delegate to `embedding/`. This ensures:
|
||||
- Existing test imports still work
|
||||
- Third-party scripts still work
|
||||
- Gradual migration path
|
||||
|
||||
**Impact:** 4 test files continue to work without import changes on the test side (shim handles it). However, to clean up tech debt, tests should be migrated directly to `embedding/`.
|
||||
|
||||
### D5: `chunker.py` stays in `memory/`, imported directly
|
||||
|
||||
**Decision:** `chunk_fulltext` remains in `memory/chunker.py`. Consumers that currently import it through `memory/vector_db.py` (like `worker/asset_index.py:456`) must switch to importing directly from `memory/chunker.py`.
|
||||
|
||||
### D6: Plugin files unchanged
|
||||
|
||||
**Decision:** Zero changes to `plugin/main.js`, `plugin/styles.css`, `plugin/manifest.json`.
|
||||
|
||||
## Complete Consumer Map
|
||||
|
||||
All 11 places that import from the files being deprecated:
|
||||
|
||||
### Importing FROM `memory/vector_db` (5 files):
|
||||
|
||||
| # | File | Line | Importing | Action |
|
||||
|---|------|------|-----------|--------|
|
||||
| 1 | `commands/embed.py` | 11-19 | `delete_paper_vectors, embed_paper, get_collection, get_embed_status, get_vector_db_path, mark_vector_build_state, read_vector_build_state` | Change to `embedding` |
|
||||
| 2 | `commands/retrieve.py` | 10 | `retrieve_chunks` | Change to `embedding` |
|
||||
| 3 | `worker/asset_index.py` | 454 | `_read_plugin_settings, chunk_fulltext, embed_paper, get_vector_db_path` | Change: `_read_plugin_settings` → `embedding._config`, `chunk_fulltext` → `memory.chunker` |
|
||||
| 4 | `memory/runtime_health.py` | 121 | `get_vector_db_path, read_vector_build_state` | Change to `embedding` |
|
||||
| 5 | `worker/vector_db.py` | 59 | `get_vector_db_path` | (shim → `embedding` via shim) |
|
||||
|
||||
### Importing FROM `worker/vector_db` (2 files):
|
||||
|
||||
| # | File | Line | Importing | Action |
|
||||
|---|------|------|-----------|--------|
|
||||
| 6 | `commands/embed.py` | 21 | `_preflight_check` | Change to `embedding.preflight` |
|
||||
| 7 | `commands/retrieve.py` | 20 | `get_embed_status` | Change to `embedding` |
|
||||
|
||||
### Importing FROM `memory/state_snapshot` (4 files — UNCHANGED):
|
||||
|
||||
| # | File | Line | Importing | Action |
|
||||
|---|------|------|-----------|--------|
|
||||
| 8 | `commands/embed.py` | 10 | `write_vector_runtime` | UNCHANGED |
|
||||
| 9 | `commands/memory.py` | 66 | `write_memory_runtime` | UNCHANGED |
|
||||
| 10 | `commands/runtime_health.py` | 9 | `write_runtime_health` | UNCHANGED |
|
||||
| 11 | `worker/status.py` | 26 | `write_memory_runtime` | UNCHANGED |
|
||||
|
||||
### Test files (4 files):
|
||||
|
||||
| # | File | Line | Importing | Action |
|
||||
|---|------|------|-----------|--------|
|
||||
| 12 | `tests/unit/memory/test_vector_db.py` | 3 | `memory.vector_db` | Change to `embedding` OR keep (shim works) |
|
||||
| 13 | `tests/unit/commands/test_embed.py` | 7 | `memory.vector_db` | Change to `embedding` OR keep (shim works) |
|
||||
| 14 | `tests/unit/commands/test_retrieve.py` | 14 | patches `worker.vector_db.get_embed_status` | Change patch target to `embedding.status` |
|
||||
| 15 | `tests/unit/worker/test_vector_db.py` | 6 | `worker.vector_db` | Change to `embedding` OR keep (shim works) |
|
||||
|
||||
### Cross-reference: what `_vec_auto_embed_if_new` in asset_index.py needs
|
||||
|
||||
Current code:
|
||||
```python
|
||||
from paperforge.memory.vector_db import (
|
||||
_read_plugin_settings,
|
||||
chunk_fulltext,
|
||||
embed_paper,
|
||||
get_vector_db_path,
|
||||
)
|
||||
```
|
||||
|
||||
After refactoring:
|
||||
```python
|
||||
from paperforge.embedding._config import _read_plugin_settings
|
||||
from paperforge.embedding import embed_paper, get_vector_db_path
|
||||
from paperforge.memory.chunker import chunk_fulltext
|
||||
```
|
||||
|
||||
## Dependency Changes
|
||||
|
||||
| Dependency | Before | After | Reason |
|
||||
|---|---|---|---|
|
||||
| `chromadb` | Required for local + API store | Required for API store only | Still needed as vector store |
|
||||
| `sentence-transformers` | Required for local mode | **REMOVED** | Replaced by API |
|
||||
| `openai` | Required for API mode | Required (only mode) | Sole embedding source |
|
||||
| `torch` (transitive) | Required via ST | **REMOVED** | Freedom from PyTorch |
|
||||
| `transformers` (transitive) | Required via ST | **REMOVED** | Freedom from HF ecosystem |
|
||||
|
||||
`pyproject.toml` extras:
|
||||
- `[vector]` extra: remove `sentence-transformers`, keep `chromadb` + `openai`
|
||||
- New package weight: `chromadb` (~30MB) + `openai` (~200KB) = ~30MB (was ~2GB)
|
||||
|
||||
## Snapshot Schema (unchanged)
|
||||
|
||||
### `memory-runtime-state.json`
|
||||
```json
|
||||
{"schema_version": 1, "paper_count_db": 650, "fresh": true, ...}
|
||||
```
|
||||
Still written by `state_snapshot.py:write_memory_runtime()`.
|
||||
|
||||
### `vector-runtime-state.json`
|
||||
```json
|
||||
{"schema_version": 1, "enabled": true, "mode": "api", "chunk_count": 1200, ...}
|
||||
```
|
||||
Still written by `state_snapshot.py:write_vector_runtime()`.
|
||||
|
||||
### `runtime-health.json`
|
||||
```json
|
||||
{"summary": {"status": "ok"}, "layers": {...}, "capabilities": {...}}
|
||||
```
|
||||
Still written by `state_snapshot.py:write_runtime_health()`.
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- FastAPI / HTTP server
|
||||
- Plugin source restructuring (main.js stays as-is)
|
||||
- Unified runtime state
|
||||
- Performance optimization of FTS
|
||||
207
docs/superpowers/specs/2026-05-18-vector-status-health-design.md
Normal file
207
docs/superpowers/specs/2026-05-18-vector-status-health-design.md
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# Vector Status + Health + Auto-Embed — Design Spec
|
||||
|
||||
> **Status:** Draft | **Date:** 2026-05-18
|
||||
> **Review:** Pending
|
||||
> **Depends on:** memory-layer-simplify (embedding/ package already extracted)
|
||||
|
||||
## Goal
|
||||
|
||||
1. **Per-paper `vector_status` column** in `paperforge.db` for fast resume, accurate health check, and agent context
|
||||
2. **`_check_vector()` strictness** — verify actual data, not just directory existence
|
||||
3. **Resume logic fix** — use SQLite instead of ChromaDB for skip detection; handle corrupted index gracefully
|
||||
4. **Auto-embed after OCR** — `embed_paper` + `vector_status` update runs when OCR completes
|
||||
|
||||
## Schema Change
|
||||
|
||||
### papers table — new column
|
||||
|
||||
```sql
|
||||
ALTER TABLE papers ADD COLUMN vector_status TEXT NOT NULL DEFAULT '';
|
||||
```
|
||||
|
||||
Values:
|
||||
| Value | Meaning |
|
||||
| ---------- | ----------------------------------------- |
|
||||
| `''` (empty) | Not OCR-done, or not yet checked |
|
||||
| `pending` | OCR done, queued for embedding |
|
||||
| `embedded` | Successfully embedded in ChromaDB |
|
||||
| `failed` | Embedding errored (retry on next --resume) |
|
||||
|
||||
### Schema version: 2 → 3
|
||||
|
||||
```python
|
||||
CURRENT_SCHEMA_VERSION = 3 # Bump from 2 for vector_status column
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
OCR completion
|
||||
↓
|
||||
ocr.py → sync → _vec_auto_embed_if_new()
|
||||
↓ ↓
|
||||
embed_paper() paperforge.db
|
||||
↓ (ChromaDB) UPDATE vector_status='embedded'
|
||||
chunks stored (or 'failed' on error)
|
||||
|
||||
embed build --resume
|
||||
↓
|
||||
SELECT zotero_key FROM papers
|
||||
WHERE ocr_status='done'
|
||||
AND (vector_status IS NULL OR vector_status != 'embedded')
|
||||
↓
|
||||
chunk → embed → UPDATE vector_status='embedded'
|
||||
|
||||
embed build --force
|
||||
↓
|
||||
DELETE vectors/ directory
|
||||
Mark all OCR-done papers as 'pending'
|
||||
For each: chunk → embed → UPDATE vector_status='embedded'
|
||||
|
||||
_check_vector()
|
||||
↓
|
||||
SELECT COUNT(*) FROM papers
|
||||
WHERE ocr_status='done' AND vector_status='embedded'
|
||||
SELECT COUNT(*) FROM papers WHERE ocr_status='done'
|
||||
↓
|
||||
coverage = embedded / total
|
||||
→ degraded if < 1.0 or corrupted or disabled
|
||||
```
|
||||
|
||||
## File Changes
|
||||
|
||||
### Part A: Schema + Builder
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/schema.py`
|
||||
- `CURRENT_SCHEMA_VERSION = 3`
|
||||
- Add `vector_status TEXT NOT NULL DEFAULT ''` to `CREATE_PAPERS`
|
||||
- Modify: `paperforge/memory/_columns.py`
|
||||
- Add `"vector_status"` to `PAPER_COLUMNS` (so rebuild includes it)
|
||||
- Modify: `paperforge/memory/builder.py`
|
||||
- Before DELETE: save old `{zotero_key: vector_status}` map
|
||||
- After INSERT: restore saved map for matching keys
|
||||
- Keeps auto-embedded status across memory rebuilds
|
||||
|
||||
### Part B: Embed Builder — Per-paper status tracking
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/embedding/builder.py`
|
||||
- `embed_paper()`: accept optional `conn` parameter to update DB
|
||||
- Or: caller handles the DB write
|
||||
- Modify: `paperforge/commands/embed.py`
|
||||
- Before build loop: `UPDATE papers SET vector_status='pending' WHERE ocr_status='done'`
|
||||
- After each paper success: `UPDATE papers SET vector_status='embedded' WHERE zotero_key=?`
|
||||
- On error: `UPDATE papers SET vector_status='failed' WHERE zotero_key=?`
|
||||
- Resume mode: `SELECT zotero_key FROM papers WHERE ocr_status='done' AND (vector_status IS NULL OR vector_status != 'embedded')`
|
||||
- Remove the ChromaDB `collection.get()` per-paper check
|
||||
|
||||
### Part C: Health — `_check_vector()` strict
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/runtime_health.py:120-153`
|
||||
- Disabled → `degraded` (was `ok`)
|
||||
- Query paperforge.db for coverage ratio
|
||||
- If coverage == 0 and no build → `degraded` "not built"
|
||||
- If 0 < coverage < 1 → `degraded` "partial coverage (N/M)"
|
||||
- If `get_embed_status().corrupted == True` → `degraded` with `repair_command: "paperforge embed build --force"`
|
||||
- Only return `ok` when: coverage == 1 AND ChromaDB healthy
|
||||
|
||||
### Part D: Auto-embed — Update status after OCR
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/worker/asset_index.py:479-505`
|
||||
- After `embed_paper()` succeeds: open paperforge.db, `UPDATE papers SET vector_status='embedded'`
|
||||
- On exception: `UPDATE papers SET vector_status='failed'`
|
||||
- Import `get_connection`, `get_memory_db_path` for DB access
|
||||
|
||||
### Part E: Resume — SQLite query instead of ChromaDB check
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/commands/embed.py:172-179`
|
||||
- Remove: `get_collection(vault).get(where={"paper_id": key})`
|
||||
- Replace with: the `SELECT` before the loop filters out already-embedded papers
|
||||
- No per-paper ChromaDB queries
|
||||
|
||||
### Part F: Agent Context — Coverage reporting
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/memory/context.py`
|
||||
- Add vector coverage stats to `get_agent_context()` output
|
||||
- `"vector": { "enabled": bool, "embedded": N, "total_ocr_done": M, "coverage": float }`
|
||||
|
||||
### Part G: Error handling — Corrupted index fallback
|
||||
|
||||
**Files:**
|
||||
- Modify: `paperforge/commands/embed.py`
|
||||
- In resume mode, if `get_embed_status().corrupted == True`: log warning, fall through to embed anyway (cannot skip, ChromaDB is broken)
|
||||
- The `--resume` loop doesn't try `collection.get()` anymore (Part E), so corruption doesn't block it
|
||||
- But `collection.add()` will still fail with compaction error → user sees clear message to use `--force`
|
||||
|
||||
## ChromaDB Corruption Recovery Flow
|
||||
|
||||
```
|
||||
User runs embed build --resume
|
||||
→ SQLite says 200 papers need embedding
|
||||
→ First embed_paper() → collection.add() → compaction error
|
||||
→ builder.py catches HNSW/compaction error
|
||||
→ Raises RuntimeError with "Run 'paperforge embed build --force'"
|
||||
→ Embed build exits with error message
|
||||
→ _check_vector() shows degraded + repair_command
|
||||
→ User runs embed build --force
|
||||
→ Delete vectors/ directory
|
||||
→ Fresh build succeeds
|
||||
```
|
||||
|
||||
## Dataset Flows
|
||||
|
||||
### Normal flow: new paper added
|
||||
```
|
||||
sync → formal-library.json updated
|
||||
→ _vec_auto_embed_if_new() chunks + embeds + sets vector_status='embedded'
|
||||
→ memory build (if user runs it) preserves vector_status
|
||||
```
|
||||
|
||||
### Normal flow: embed build on all papers
|
||||
```
|
||||
embed build --force
|
||||
→ DELETE vectors/
|
||||
→ SQL UPDATE vector_status='pending' WHERE ocr_status='done'
|
||||
→ For each: embed + UPDATE vector_status='embedded'
|
||||
→ _check_vector() sees 658/658 → ok
|
||||
```
|
||||
|
||||
### Normal flow: incremental update
|
||||
```
|
||||
embed build --resume
|
||||
→ SQL SELECT WHERE ocr_status='done' AND vector_status!='embedded'
|
||||
→ Only new or failed papers
|
||||
→ Fast: no ChromaDB queries
|
||||
```
|
||||
|
||||
### Degraded flow: partial build
|
||||
```
|
||||
embed build interrupted after 300/658
|
||||
→ _check_vector() sees 300/658 → degraded "partial (58%)"
|
||||
→ Agent sees coverage < 1.0, falls back to FTS
|
||||
→ Next embed build --resume picks up remaining 358
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Unit tests
|
||||
- `tests/unit/memory/test_schema.py`: schema v3 has vector_status column
|
||||
- `tests/unit/commands/test_embed.py`: resume uses SQLite, not ChromaDB get
|
||||
- `tests/unit/memory/test_health.py`: _check_vector returns degraded when coverage < 1
|
||||
|
||||
### Manual tests
|
||||
- `paperforge embed build --resume` after partial build
|
||||
- `paperforge embed build --force`
|
||||
- `paperforge runtime-health --json` shows correct vector coverage
|
||||
- `paperforge agent-context --json` shows vector coverage
|
||||
|
||||
## Out of Scope
|
||||
|
||||
- Automatic embed build trigger on OCR completion (user still clicks Build or runs CLI)
|
||||
- UI for per-paper vector status in plugin dashboard
|
||||
- ChromaDB version pinning/downgrade (user manages chromadb version)
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"id": "paperforge",
|
||||
"name": "PaperForge",
|
||||
"version": "1.5.6rc3",
|
||||
"version": "1.5.6rc4",
|
||||
"minAppVersion": "1.9.0",
|
||||
"description": "Zotero literature pipeline for Obsidian. Sync PDFs, run OCR, and read with AI-assisted deep reading.",
|
||||
"author": "Lin Zhaoxuan",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
"""paperforge — PaperForge package."""
|
||||
|
||||
__version__ = "1.5.6rc3"
|
||||
__version__ = "1.5.6rc4"
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ def run(args: argparse.Namespace) -> int:
|
|||
chunk_count=status.get("chunk_count", 0),
|
||||
build_state=status.get("build_state"),
|
||||
healthy=status.get("healthy", True),
|
||||
corrupted=status.get("corrupted", False),
|
||||
error=status.get("error", ""),
|
||||
)
|
||||
|
||||
|
|
@ -176,7 +177,10 @@ def run(args: argparse.Namespace) -> int:
|
|||
if existing and existing.get("ids") and len(existing["ids"]) > 0:
|
||||
papers_skipped += 1
|
||||
continue
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
err = str(exc).lower()
|
||||
if "hnsw" in err or "compaction" in err:
|
||||
logger.warning("ChromaDB index corrupted — rebuilding from scratch. Use --force next time for clean rebuild.")
|
||||
pass
|
||||
chunks = chunk_fulltext(fulltext_path)
|
||||
if not chunks:
|
||||
|
|
|
|||
|
|
@ -28,10 +28,20 @@ def embed_paper(vault: Path, zotero_key: str, chunks: list[dict]) -> int:
|
|||
]
|
||||
|
||||
embeddings = provider.encode(texts)
|
||||
collection.add(
|
||||
ids=ids,
|
||||
embeddings=embeddings,
|
||||
documents=texts,
|
||||
metadatas=metadatas,
|
||||
)
|
||||
try:
|
||||
collection.add(
|
||||
ids=ids,
|
||||
embeddings=embeddings,
|
||||
documents=texts,
|
||||
metadatas=metadatas,
|
||||
)
|
||||
except Exception as exc:
|
||||
err = str(exc).lower()
|
||||
if "hnsw" in err or "compaction" in err or "segment" in err:
|
||||
raise RuntimeError(
|
||||
f"ChromaDB index error (possibly corrupted). "
|
||||
f"Run 'paperforge embed build --force' to rebuild from scratch. "
|
||||
f"Original error: {exc}"
|
||||
) from exc
|
||||
raise
|
||||
return len(chunks)
|
||||
|
|
|
|||
|
|
@ -12,13 +12,14 @@ logger = logging.getLogger(__name__)
|
|||
def get_embed_status(vault: Path) -> dict:
|
||||
"""Get vector DB status. API-only mode.
|
||||
|
||||
Returns dict with keys: db_exists, chunk_count, model, mode, healthy, error.
|
||||
Returns dict with keys: db_exists, chunk_count, model, mode, healthy, error, corrupted.
|
||||
"""
|
||||
db_path = get_vector_db_path(vault)
|
||||
exists = db_path.exists()
|
||||
chunk_count = 0
|
||||
healthy = True
|
||||
error = ""
|
||||
corrupted = False
|
||||
if exists:
|
||||
try:
|
||||
collection = get_collection(vault)
|
||||
|
|
@ -26,6 +27,8 @@ def get_embed_status(vault: Path) -> dict:
|
|||
except Exception as exc:
|
||||
healthy = False
|
||||
error = str(exc)
|
||||
err_lower = str(exc).lower()
|
||||
corrupted = "hnsw" in err_lower or "corrupt" in err_lower
|
||||
|
||||
model = get_api_model(vault)
|
||||
|
||||
|
|
@ -35,5 +38,6 @@ def get_embed_status(vault: Path) -> dict:
|
|||
"model": model,
|
||||
"mode": "api",
|
||||
"healthy": healthy,
|
||||
"corrupted": corrupted,
|
||||
"error": error,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,8 @@ def write_memory_runtime(vault: Path, *, paper_count_db: int,
|
|||
def write_vector_runtime(vault: Path, *, enabled: bool, mode: str, model: str,
|
||||
deps_installed: bool, deps_missing: list[str] | None,
|
||||
py_version: str, db_exists: bool, chunk_count: int,
|
||||
build_state: dict | None) -> None:
|
||||
build_state: dict | None, healthy: bool = True,
|
||||
error: str = "", corrupted: bool = False) -> None:
|
||||
snap = {
|
||||
"schema_version": 1,
|
||||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||
|
|
@ -51,6 +52,9 @@ def write_vector_runtime(vault: Path, *, enabled: bool, mode: str, model: str,
|
|||
"py_version": py_version,
|
||||
"db_exists": db_exists,
|
||||
"chunk_count": chunk_count,
|
||||
"healthy": healthy,
|
||||
"corrupted": corrupted,
|
||||
"error": error,
|
||||
"build_state": build_state or {},
|
||||
}
|
||||
path = _snapshot_dir(vault) / "vector-runtime-state.json"
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ const memoryState = (() => {
|
|||
if (!s.enabled) return false;
|
||||
if (!s.deps_installed) return false;
|
||||
if (!s.db_exists) return false;
|
||||
if (s.healthy === false) return false;
|
||||
if (s.chunk_count === 0) return false;
|
||||
return true;
|
||||
}
|
||||
|
|
@ -111,6 +112,7 @@ const memoryState = (() => {
|
|||
function getVectorStatusText(vaultPath) {
|
||||
var s = getVectorRuntime(vaultPath);
|
||||
if (!s) return 'Status unavailable';
|
||||
if (s.healthy === false) return 'Vector index unreadable - rebuild required';
|
||||
return 'Chunks: ' + s.chunk_count + ' | ' + s.model + ' | ' + s.mode;
|
||||
}
|
||||
|
||||
|
|
@ -374,6 +376,20 @@ function runSubprocess(pythonExe, args, cwd, timeout, _spawn, env) {
|
|||
|
||||
}
|
||||
|
||||
function getDisclosureState(store, key, defaultCollapsed) {
|
||||
if (!store || typeof store !== 'object') return !!defaultCollapsed;
|
||||
if (!Object.prototype.hasOwnProperty.call(store, key)) return !!defaultCollapsed;
|
||||
return !!store[key];
|
||||
}
|
||||
|
||||
function toggleDisclosureState(store, key, defaultCollapsed) {
|
||||
const next = !getDisclosureState(store, key, defaultCollapsed);
|
||||
if (store && typeof store === 'object') {
|
||||
store[key] = next;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
// ── Cross-platform Python and BBT detection (macOS/Linux) ──
|
||||
|
||||
let _gitDir = null;
|
||||
|
|
@ -671,22 +687,12 @@ Object.assign(LANG.en, {
|
|||
feat_skills_system: 'System Skills ship with PaperForge and are updated alongside PaperForge.',
|
||||
feat_skills_user: 'User Skills are custom skills you install from community or create yourself.',
|
||||
feat_memory_desc: 'The Memory Layer is the core data engine of PaperForge, powered by SQLite. It integrates all literature metadata (papers, assets, aliases, reading events), provides FTS5 full-text search across titles/abstracts/authors/collections, and enables the agent-context and paper-status commands. Always active — no toggle needed.',
|
||||
feat_vector_desc: 'Vector Database enables semantic search across OCR-extracted fulltext using embedding models. Documents are split into chunks, embedded into vector space, and stored in ChromaDB. Supports local models (free, CPU) or OpenAI API (paid, faster).',
|
||||
feat_vector_config_label: 'Advanced Configuration',
|
||||
feat_vector_desc: 'Vector Database enables semantic search across OCR-extracted fulltext via API embedding. Documents are split into chunks, embedded via OpenAI-compatible API, and stored in ChromaDB.',
|
||||
feat_vector_config_label: 'Vector Settings',
|
||||
feat_agent_platform: 'Agent Platform',
|
||||
feat_agent_platform_desc: 'Select which agent platform to manage skills for.',
|
||||
feat_vector_enable: 'Enable Vector Retrieval',
|
||||
feat_vector_enable_desc: 'Semantic search across OCR fulltext. Requires: pip install chromadb sentence-transformers openai (~500MB).',
|
||||
feat_hf_mirror: 'HF Mirror / Endpoint',
|
||||
feat_hf_mirror_desc: 'Model download source. Try official if mirror fails. Custom: type any URL.',
|
||||
feat_custom_endpoint: 'Custom Endpoint',
|
||||
feat_custom_endpoint_desc: 'Enter a custom HuggingFace mirror URL.',
|
||||
feat_hf_token: 'HF Token',
|
||||
feat_hf_token_desc: 'HuggingFace access token (optional, helps with rate limits and gated models).',
|
||||
feat_model: 'Model',
|
||||
feat_embed_mode: 'Embedding Mode',
|
||||
feat_embed_mode_local: 'Local (free, CPU)',
|
||||
feat_embed_mode_api: 'API (OpenAI, paid)',
|
||||
feat_vector_enable_desc: 'Semantic search across OCR fulltext. Requires: pip install openai chromadb (~35MB).',
|
||||
feat_openai_key: 'OpenAI API Key',
|
||||
feat_openai_key_desc: 'Used for API embedding calls. Model is defined below.',
|
||||
feat_verify: 'Verify',
|
||||
|
|
@ -695,23 +701,18 @@ Object.assign(LANG.en, {
|
|||
feat_rebuild_vectors_desc: 'Rebuild all OCR fulltext vectors. Required after model or mode change.',
|
||||
feat_rebuild_vectors_changed: 'Model changed — rebuild to update all vectors.',
|
||||
feat_install_deps: 'Install Dependencies',
|
||||
feat_install_deps_desc: 'pip install chromadb sentence-transformers openai (~500MB).',
|
||||
feat_install_deps_desc_api: 'pip install openai (~5MB).',
|
||||
feat_install_deps_desc_local: 'pip install chromadb sentence-transformers openai (~500MB).',
|
||||
feat_model_bge_small: 'Best balance — fast, accurate, recommended for most users (384d, 130MB)',
|
||||
feat_model_minilm: 'Lightest & fastest — lower accuracy, minimal disk (384d, 80MB)',
|
||||
feat_model_bge_base: 'Highest accuracy — slower, large disk footprint (768d, 440MB)',
|
||||
feat_install_deps_desc: 'pip install chromadb openai (~35MB).',
|
||||
feat_api_base_url: 'API Base URL',
|
||||
feat_api_base_url_desc: 'Custom OpenAI-compatible API endpoint. Leave empty for default.',
|
||||
feat_api_model: 'API Model',
|
||||
feat_api_model_desc: 'Embedding model name for this endpoint.',
|
||||
feat_deps_missing: 'Dependencies not installed. Required: chromadb, sentence-transformers, openai.',
|
||||
feat_deps_missing_api: 'Dependencies not installed. Required: openai.',
|
||||
feat_deps_missing_local: 'Dependencies not installed. Required: chromadb, sentence-transformers, openai.',
|
||||
feat_deps_missing: 'Dependencies not installed. Required: chromadb, openai.',
|
||||
feat_deps_checking: 'Checking dependencies...',
|
||||
feat_no_python: 'No Python found. Check Installation tab.',
|
||||
feat_rebuild_btn: 'Rebuild',
|
||||
feat_build_btn: 'Build',
|
||||
feat_vector_corrupted: 'Vector index corrupted — needs force rebuild.',
|
||||
feat_vector_rebuild_force_btn: 'Force Rebuild',
|
||||
feat_building: 'Building...',
|
||||
feat_installing: 'Installing...',
|
||||
feat_install_btn: 'Install',
|
||||
|
|
@ -784,21 +785,12 @@ Object.assign(LANG.zh, {
|
|||
feat_skills_user: '用户技能是你自行安装或创建的自定义技能。',
|
||||
feat_memory_desc: '记忆层是 PaperForge 的核心数据引擎，基于 SQLite 构建。它整合了所有文献元数据（论文、资源文件、别名、阅读事件），支持 FTS5 全文检索（可搜索标题、摘要、作者、分类），并为 agent-context 和 paper-status 命令提供数据支撑。始终运行，无需手动开启。',
|
||||
feat_vector_desc: '向量数据库通过嵌入模型实现 OCR 全文的语义搜索。文档被切分为文本块（chunk），编码为向量存入 ChromaDB。支持本地模型（免费，CPU 运行）或 OpenAI API（付费，更快速）。',
|
||||
feat_vector_config_label: '高级配置',
|
||||
feat_vector_config_label: '向量库配置',
|
||||
feat_agent_platform: 'Agent 平台',
|
||||
feat_agent_platform_desc: '选择要管理的 Agent 平台。',
|
||||
feat_vector_enable: '启用向量检索',
|
||||
feat_vector_enable_desc: '对 OCR 全文进行语义搜索。需安装: pip install chromadb sentence-transformers openai (~500MB)。',
|
||||
feat_hf_mirror: 'HF 镜像站 / 端点',
|
||||
feat_hf_mirror_desc: '模型下载源。镜像不可用时尝试官方源。自定义：输入任意 URL。',
|
||||
feat_custom_endpoint: '自定义端点',
|
||||
feat_custom_endpoint_desc: '输入自定义 HuggingFace 镜像 URL。',
|
||||
feat_hf_token: 'HF Token',
|
||||
feat_hf_token_desc: 'HuggingFace 访问令牌（可选，有助于解除限速和下载受限模型）。',
|
||||
feat_model: '模型',
|
||||
feat_embed_mode: '嵌入模式',
|
||||
feat_embed_mode_local: '本地（免费，CPU）',
|
||||
feat_embed_mode_api: 'API（OpenAI，付费）',
|
||||
feat_openai_key: 'OpenAI API Key',
|
||||
feat_openai_key_desc: '用于 API 嵌入调用，模型在下方定义。',
|
||||
feat_verify: '验证',
|
||||
|
|
@ -807,23 +799,17 @@ Object.assign(LANG.zh, {
|
|||
feat_rebuild_vectors_desc: '重建所有 OCR 全文向量。更换模型或模式后需要重建。',
|
||||
feat_rebuild_vectors_changed: '模型已更换 — 需要重建向量。',
|
||||
feat_install_deps: '安装依赖',
|
||||
feat_install_deps_desc: 'pip install chromadb sentence-transformers openai (~500MB)。',
|
||||
feat_install_deps_desc_api: 'pip install openai (~5MB)。',
|
||||
feat_install_deps_desc_local: 'pip install chromadb sentence-transformers openai (~500MB)。',
|
||||
feat_model_bge_small: '最佳平衡 — 快速、准确，推荐大多数用户使用 (384d, 130MB)',
|
||||
feat_model_minilm: '最轻最快 — 精度略低，磁盘占用最小 (384d, 80MB)',
|
||||
feat_model_bge_base: '最高精度 — 较慢，磁盘占用大 (768d, 440MB)',
|
||||
feat_api_base_url: 'API 地址',
|
||||
feat_api_base_url_desc: '自定义 OpenAI 兼容 API 端点。留空使用默认地址。',
|
||||
feat_api_model: 'API 模型',
|
||||
feat_api_model_desc: '该端点使用的嵌入模型名称。',
|
||||
feat_deps_missing: '依赖未安装。需要：chromadb, sentence-transformers, openai。',
|
||||
feat_deps_missing_api: '依赖未安装。需要：openai。',
|
||||
feat_deps_missing_local: '依赖未安装。需要：chromadb, sentence-transformers, openai。',
|
||||
feat_deps_missing: '依赖未安装。需要：chromadb, openai。',
|
||||
feat_deps_checking: '正在检测依赖…',
|
||||
feat_no_python: '未找到 Python。请查看安装标签页。',
|
||||
feat_rebuild_btn: '重建',
|
||||
feat_build_btn: '构建',
|
||||
feat_vector_corrupted: '向量索引已损坏 — 需要强制重建。',
|
||||
feat_vector_rebuild_force_btn: '强制重建',
|
||||
feat_building: '构建中…',
|
||||
feat_installing: '安装中…',
|
||||
feat_install_btn: '安装',
|
||||
|
|
@ -885,14 +871,9 @@ const DEFAULT_SETTINGS = {
|
|||
vector_db: false,
|
||||
},
|
||||
selected_skill_platform: 'opencode',
|
||||
vector_db_mode: 'api',
|
||||
vector_db_model: 'BAAI/bge-small-en-v1.5',
|
||||
vector_db_api_key: '',
|
||||
vector_db_api_base: '',
|
||||
vector_db_api_model: 'text-embedding-3-small',
|
||||
vector_db_hf_endpoint: 'https://hf-mirror.com',
|
||||
vector_db_hf_token: '',
|
||||
vector_db_last_model: '',
|
||||
frozen_skills: {},
|
||||
};
|
||||
|
||||
|
|
@ -2592,6 +2573,7 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
this._vectorDepsOk = null; // null = not checked, bool = cached
|
||||
this._embedStatusText = null;
|
||||
this._skillsCollapsed = { user: true }; // User skills collapsed by default
|
||||
this._featurePanelsCollapsed = {};
|
||||
this.activeTab = 'setup';
|
||||
}
|
||||
|
||||
|
|
@ -3205,21 +3187,23 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
}
|
||||
|
||||
// --- Section: Advanced (Memory Layer + Vector DB, collapsed by default) ---
|
||||
if (this._advCollapsed === undefined) this._advCollapsed = true;
|
||||
const advHeader = containerEl.createEl('div', { cls: 'paperforge-collapsible-header' });
|
||||
advHeader.style.cssText = 'cursor:pointer; display:flex; align-items:center; gap:8px; padding:8px 0; user-select:none;';
|
||||
const advArrow = advHeader.createEl('span', { text: '\u25B6', cls: 'paperforge-collapsible-arrow' });
|
||||
advArrow.style.cssText = 'display:inline-block; transition:transform 0.2s; font-size:10px; transform:rotate(90deg);';
|
||||
advHeader.createEl('h3', { text: 'Advanced', cls: 'paperforge-collapsible-title' });
|
||||
advHeader.createEl('span', { text: 'Memory + Vector DB + Embedding', cls: 'paperforge-collapsible-sub' });
|
||||
advHeader.lastChild.style.cssText = 'font-size:11px; color:var(--text-muted); margin-left:8px;';
|
||||
advArrow.style.cssText = 'display:inline-block; transition:transform 0.2s; font-size:10px; transform:' + (this._advCollapsed ? 'rotate(0deg)' : 'rotate(90deg)') + ';';
|
||||
const advTitle = advHeader.createEl('span', { text: 'Advanced' });
|
||||
advTitle.style.cssText = 'font-size:16px; font-weight:700; line-height:1.4;';
|
||||
const advSub = advHeader.createEl('span', { text: 'Memory + Vector DB + Embedding' });
|
||||
advSub.style.cssText = 'font-size:12px; color:var(--text-muted); margin-left:10px;';
|
||||
|
||||
const advContent = containerEl.createEl('div', { cls: 'paperforge-collapsible-content' });
|
||||
advContent.style.display = 'none';
|
||||
advContent.style.display = this._advCollapsed ? 'none' : '';
|
||||
|
||||
advHeader.addEventListener('click', () => {
|
||||
const collapsed = advContent.style.display === 'none';
|
||||
advContent.style.display = collapsed ? '' : 'none';
|
||||
advArrow.style.transform = collapsed ? 'rotate(0deg)' : 'rotate(90deg)';
|
||||
this._advCollapsed = !this._advCollapsed;
|
||||
advContent.style.display = this._advCollapsed ? 'none' : '';
|
||||
advArrow.style.transform = this._advCollapsed ? 'rotate(0deg)' : 'rotate(90deg)';
|
||||
});
|
||||
|
||||
// Memory Layer section (inside Advanced)
|
||||
|
|
@ -3285,11 +3269,16 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
vecConfigHeader.createEl('span', { text: t('feat_vector_config_label'), cls: '' }).style.cssText = 'font-size:12px; color:var(--text-muted);';
|
||||
const vecConfigContent = containerEl.createEl('div', { cls: 'paperforge-vector-config' });
|
||||
|
||||
let vecConfigCollapsed = false;
|
||||
const applyVectorConfigDisclosure = (collapsed) => {
|
||||
vecConfigContent.style.display = collapsed ? 'none' : '';
|
||||
vecArrow.style.transform = collapsed ? 'rotate(-90deg)' : 'rotate(0deg)';
|
||||
};
|
||||
|
||||
applyVectorConfigDisclosure(getDisclosureState(this._featurePanelsCollapsed, 'vectorConfig', false));
|
||||
|
||||
vecConfigHeader.addEventListener('click', () => {
|
||||
vecConfigCollapsed = !vecConfigCollapsed;
|
||||
vecConfigContent.style.display = vecConfigCollapsed ? 'none' : '';
|
||||
vecArrow.style.transform = vecConfigCollapsed ? 'rotate(-90deg)' : 'rotate(0deg)';
|
||||
const collapsed = toggleDisclosureState(this._featurePanelsCollapsed, 'vectorConfig', false);
|
||||
applyVectorConfigDisclosure(collapsed);
|
||||
});
|
||||
|
||||
// === Resolve state ===
|
||||
|
|
@ -3312,61 +3301,7 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
}
|
||||
}
|
||||
|
||||
_renderHfMirror(containerEl) {
|
||||
const setting = new Setting(containerEl)
|
||||
.setName(t('feat_hf_mirror'))
|
||||
.setDesc(t('feat_hf_mirror_desc'))
|
||||
.addDropdown(dropdown => {
|
||||
dropdown.addOption('https://hf-mirror.com', 'hf-mirror.com (recommended)');
|
||||
dropdown.addOption('https://huggingface.co', 'huggingface.co (official)');
|
||||
dropdown.addOption('__custom__', 'Custom...');
|
||||
const current = this.plugin.settings.vector_db_hf_endpoint || 'https://hf-mirror.com';
|
||||
const isPreset = ['https://hf-mirror.com', 'https://huggingface.co'].includes(current);
|
||||
dropdown.setValue(isPreset ? current : '__custom__')
|
||||
.onChange(value => {
|
||||
if (value !== '__custom__') {
|
||||
this.plugin.settings.vector_db_hf_endpoint = value;
|
||||
this.plugin.saveSettings();
|
||||
if (customInput) { customInput.settingEl.style.display = 'none'; if (this._hfCustomText) this._hfCustomText.setValue(''); }
|
||||
} else {
|
||||
if (customInput) customInput.settingEl.style.display = '';
|
||||
}
|
||||
});
|
||||
});
|
||||
const customInput = new Setting(containerEl)
|
||||
.setName(t('feat_custom_endpoint'))
|
||||
.setDesc(t('feat_custom_endpoint_desc'))
|
||||
.addText(text => {
|
||||
this._hfCustomText = text;
|
||||
const current = this.plugin.settings.vector_db_hf_endpoint || '';
|
||||
const isPreset = ['https://hf-mirror.com', 'https://huggingface.co'].includes(current);
|
||||
text.setPlaceholder('https://your-mirror.com')
|
||||
.setValue(isPreset ? '' : current)
|
||||
.onChange(value => {
|
||||
this.plugin.settings.vector_db_hf_endpoint = value;
|
||||
this.plugin.saveSettings();
|
||||
});
|
||||
});
|
||||
const current = this.plugin.settings.vector_db_hf_endpoint || 'https://hf-mirror.com';
|
||||
const isPreset = ['https://hf-mirror.com', 'https://huggingface.co'].includes(current);
|
||||
if (isPreset) customInput.settingEl.style.display = 'none';
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName(t('feat_hf_token'))
|
||||
.setDesc(t('feat_hf_token_desc'))
|
||||
.addText(text => {
|
||||
text.setPlaceholder('hf_...')
|
||||
.setValue(this.plugin.settings.vector_db_hf_token || '')
|
||||
.onChange(value => {
|
||||
this.plugin.settings.vector_db_hf_token = value;
|
||||
this.plugin.saveSettings();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
_renderApiConfig(containerEl) {
|
||||
if (this.plugin.settings.vector_db_mode !== 'api') return;
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName(t('feat_openai_key'))
|
||||
.setDesc(t('feat_openai_key_desc'))
|
||||
|
|
@ -3405,12 +3340,11 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
_renderVectorNoDeps(containerEl) {
|
||||
const box = containerEl.createEl('div');
|
||||
box.style.cssText = 'padding:8px 12px; margin:8px 0; background:var(--background-secondary); border-radius:4px;';
|
||||
const modeKey = (this.plugin.settings.vector_db_mode === 'api') ? 'api' : 'local';
|
||||
box.setText(t('feat_deps_missing_' + modeKey));
|
||||
box.setText(t('feat_deps_missing'));
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName(t('feat_install_deps'))
|
||||
.setDesc(t('feat_install_deps_desc_' + modeKey))
|
||||
.setDesc(t('feat_install_deps_desc'))
|
||||
.addButton(button => {
|
||||
button.setButtonText(t('feat_install_btn'))
|
||||
.setCta()
|
||||
|
|
@ -3420,12 +3354,11 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
if (!pyResult.path) { new Notice(t('feat_no_python')); return; }
|
||||
button.setButtonText(t('feat_installing'));
|
||||
button.setDisabled(true);
|
||||
const mode = this.plugin.settings.vector_db_mode || 'api';
|
||||
const pkgs = mode === 'api' ? 'openai' : 'chromadb sentence-transformers openai';
|
||||
const pkgs = 'chromadb openai';
|
||||
const notice = new Notice(t('feat_installing_pkgs').replace('{pkgs}', pkgs), 0);
|
||||
try {
|
||||
const { execFile: execFileInstall } = require('node:child_process');
|
||||
const env = Object.assign({}, process.env, { PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1', HF_ENDPOINT: this.plugin.settings.vector_db_hf_endpoint || 'https://hf-mirror.com', HF_TOKEN: this.plugin.settings.vector_db_hf_token || '' });
|
||||
const env = Object.assign({}, process.env, { PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' });
|
||||
const pkgsArg = pkgs.split(' ');
|
||||
await new Promise((resolve, reject) => {
|
||||
execFileInstall(pyResult.path, [...pyResult.extraArgs, '-m', 'pip', 'install', ...pkgsArg], {
|
||||
|
|
@ -3454,127 +3387,7 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
statusEl.style.cssText = 'padding:8px 12px; margin:8px 0; background:var(--background-secondary); border-radius:4px;';
|
||||
statusEl.setText(memoryState.getVectorStatusText(vp));
|
||||
|
||||
// Detect model mismatch
|
||||
const embedInfo = memoryState.getVectorRuntime(vp);
|
||||
const currentModel = this._getCurrentModelKey();
|
||||
const lastModel = this.plugin.settings.vector_db_last_model || '';
|
||||
const modelChanged = embedInfo && embedInfo.db_exists && lastModel && lastModel !== currentModel;
|
||||
|
||||
if (modelChanged) {
|
||||
const warnEl = containerEl.createEl('div');
|
||||
warnEl.style.cssText = 'padding:8px 12px; margin:8px 0; background:var(--background-modifier-warning); border-radius:4px;';
|
||||
warnEl.setText(t('feat_model_changed_warn').replace('{0}', lastModel).replace('{1}', currentModel));
|
||||
}
|
||||
|
||||
// Mode selector
|
||||
new Setting(containerEl)
|
||||
.setName(t('feat_embed_mode'))
|
||||
.addDropdown(dropdown => {
|
||||
dropdown.addOption('api', t('feat_embed_mode_api'));
|
||||
dropdown.addOption('local', t('feat_embed_mode_local'));
|
||||
dropdown.setValue(this.plugin.settings.vector_db_mode)
|
||||
.onChange(value => {
|
||||
this.plugin.settings.vector_db_mode = value;
|
||||
this.plugin.saveSettings();
|
||||
// Clear cache immediately — show "checking..." then refresh in background
|
||||
this._vectorDepsOk = null;
|
||||
this._embedStatusText = null;
|
||||
this.display();
|
||||
const vp = this.app.vault.adapter.basePath;
|
||||
const py = memoryState.getCachedPython(vp, this.plugin.settings);
|
||||
const { execFile } = require('node:child_process');
|
||||
const args = [...py.extraArgs, '-m', 'paperforge', '--vault', vp, 'embed', 'status', '--json'];
|
||||
execFile(py.path, args, { cwd: vp, timeout: 15000, windowsHide: true }, () => {
|
||||
this._vectorDepsOk = null;
|
||||
this._embedStatusText = null;
|
||||
this.display();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Model selector (local mode)
|
||||
if (this.plugin.settings.vector_db_mode === 'local') {
|
||||
// HF settings only relevant for local model downloads
|
||||
this._renderHfMirror(containerEl);
|
||||
const modelDesc = {
|
||||
'BAAI/bge-small-en-v1.5': t('feat_model_bge_small'),
|
||||
'sentence-transformers/all-MiniLM-L6-v2': t('feat_model_minilm'),
|
||||
'BAAI/bge-base-en-v1.5': t('feat_model_bge_base'),
|
||||
};
|
||||
new Setting(containerEl)
|
||||
.setName(t('feat_model'))
|
||||
.setDesc(modelDesc[this.plugin.settings.vector_db_model] || '')
|
||||
.addDropdown(dropdown => {
|
||||
dropdown.addOption('BAAI/bge-small-en-v1.5', 'bge-small (384d, 130MB)');
|
||||
dropdown.addOption('sentence-transformers/all-MiniLM-L6-v2', 'MiniLM (384d, 80MB)');
|
||||
dropdown.addOption('BAAI/bge-base-en-v1.5', 'bge-base (768d, 440MB)');
|
||||
dropdown.setValue(this.plugin.settings.vector_db_model)
|
||||
.onChange(value => {
|
||||
this.plugin.settings.vector_db_model = value;
|
||||
this.plugin.saveSettings();
|
||||
this.display();
|
||||
});
|
||||
})
|
||||
.addButton(button => {
|
||||
const model = this.plugin.settings.vector_db_model;
|
||||
const cacheName = 'models--' + model.replace('/', '--');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const cachePath = path.join(os.homedir(), '.cache', 'huggingface', 'hub', cacheName);
|
||||
|
||||
// Check integrity: directory exists AND has snapshots with files
|
||||
let isCached = false;
|
||||
if (fs.existsSync(cachePath)) {
|
||||
const snapDir = path.join(cachePath, 'snapshots');
|
||||
if (fs.existsSync(snapDir)) {
|
||||
try {
|
||||
const entries = fs.readdirSync(snapDir);
|
||||
isCached = entries.some(e => {
|
||||
const p = path.join(snapDir, e);
|
||||
return fs.statSync(p).isDirectory() && fs.readdirSync(p).length > 0;
|
||||
});
|
||||
} catch (_) {}
|
||||
}
|
||||
}
|
||||
|
||||
if (isCached) {
|
||||
button.setButtonText(t('feat_uninstall_btn')).setWarning();
|
||||
} else {
|
||||
button.setButtonText(t('feat_not_cached'));
|
||||
button.setDisabled(true);
|
||||
}
|
||||
button.onClick(async () => {
|
||||
if (!isCached) return;
|
||||
button.setButtonText(t('feat_removing'));
|
||||
button.setDisabled(true);
|
||||
try {
|
||||
const pyResult = memoryState.getCachedPython(vp, this.plugin.settings);
|
||||
const { execFile: execFileRm } = require('node:child_process');
|
||||
const env = Object.assign({}, process.env, { PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' });
|
||||
await new Promise((resolve, reject) => {
|
||||
execFileRm(pyResult.path, [...pyResult.extraArgs, '-c', 'import shutil, os; p=os.path.join(os.path.expanduser("~/.cache/huggingface/hub"), "' + cacheName + '"); shutil.rmtree(p,ignore_errors=True); print("done")'], {
|
||||
cwd: vp, timeout: 30000, env: env, windowsHide: true,
|
||||
}, (error) => error ? reject(error) : resolve());
|
||||
});
|
||||
new Notice(t('feat_cache_removed'));
|
||||
} catch (e) {
|
||||
new Notice(t('feat_cache_remove_failed').replace('{0}', e.stderr || e.message || String(e)));
|
||||
}
|
||||
this.display();
|
||||
});
|
||||
});
|
||||
|
||||
// INFO: HF download notice for local mode
|
||||
const infoDiv = containerEl.createDiv({ cls: 'setting-item-description' });
|
||||
infoDiv.createEl('p', {
|
||||
text: 'Local mode downloads models from Hugging Face on first use. '
|
||||
+ 'If inaccessible, set an HF Endpoint above (e.g. https://hf-mirror.com) or switch to API mode.',
|
||||
cls: 'paperforge-settings-desc',
|
||||
});
|
||||
}
|
||||
|
||||
// API config (api mode)
|
||||
// API config
|
||||
this._renderApiConfig(containerEl);
|
||||
|
||||
// Embed build section with progress bar
|
||||
|
|
@ -3640,33 +3453,20 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
} else {
|
||||
const embedInfo = memoryState.getVectorRuntime(vp);
|
||||
const hasChunks = embedInfo && embedInfo.chunk_count > 0;
|
||||
const isCorrupted = embedInfo && embedInfo.corrupted;
|
||||
|
||||
if (hasChunks) {
|
||||
embedControls.createEl('span', {
|
||||
text: `${embedInfo.chunk_count} chunks embedded`,
|
||||
cls: 'setting-item-description'
|
||||
});
|
||||
}
|
||||
|
||||
const buildBtn = embedControls.createEl('button');
|
||||
buildBtn.setText(hasChunks ? t('feat_rebuild_btn') : t('feat_build_btn'));
|
||||
buildBtn.addClass('mod-cta');
|
||||
buildBtn.addEventListener('click', () => {
|
||||
const startBuild = (flag) => {
|
||||
const py = memoryState.getCachedPython(vp, this.plugin.settings);
|
||||
if (!py.path) { new Notice(t('feat_no_python')); return; }
|
||||
|
||||
const env = Object.assign({}, process.env, {
|
||||
PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1',
|
||||
HF_ENDPOINT: this.plugin.settings.vector_db_hf_endpoint || 'https://hf-mirror.com',
|
||||
HF_TOKEN: this.plugin.settings.vector_db_hf_token || '',
|
||||
VECTOR_DB_API_KEY: this.plugin.settings.vector_db_api_key || '',
|
||||
VECTOR_DB_API_BASE: this.plugin.settings.vector_db_api_base || '',
|
||||
VECTOR_DB_API_MODEL: this.plugin.settings.vector_db_api_model || ''
|
||||
});
|
||||
this.plugin._embedStderr = '';
|
||||
this.plugin._embedProgress = { current: 0, total: 0, key: '' };
|
||||
|
||||
this.plugin._embedProcess = this._callPython(['embed', 'build', '--resume'], {
|
||||
this.plugin._embedProcess = this._callPython(['embed', 'build', flag], {
|
||||
stream: true,
|
||||
env: env,
|
||||
onData: (data) => {
|
||||
|
|
@ -3698,7 +3498,6 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
this.plugin._embedProcess = null;
|
||||
if (code === 0) {
|
||||
this.plugin._embedProgress.current = this.plugin._embedProgress.total;
|
||||
this.plugin.settings.vector_db_last_model = currentModel;
|
||||
this.plugin.saveSettings();
|
||||
this._embedStatusText = memoryState.getVectorStatusText(vp);
|
||||
new Notice(t('feat_build_complete'));
|
||||
|
|
@ -3714,7 +3513,35 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
});
|
||||
|
||||
this.display();
|
||||
});
|
||||
}
|
||||
|
||||
// Corruption warning
|
||||
if (isCorrupted) {
|
||||
const warnEl = embedSection.createEl('div');
|
||||
warnEl.style.cssText = 'padding:8px 12px; margin:8px 0; background:var(--background-modifier-warning); border-radius:4px; font-size:12px; display:flex; align-items:center; justify-content:space-between;';
|
||||
warnEl.createEl('span', { text: t('feat_vector_corrupted') });
|
||||
const forceBtn = warnEl.createEl('button', { text: t('feat_vector_rebuild_force_btn') });
|
||||
forceBtn.className = 'mod-cta';
|
||||
forceBtn.addEventListener('click', () => startBuild('--force'));
|
||||
}
|
||||
|
||||
// Chunk count + buttons
|
||||
if (hasChunks && !isCorrupted) {
|
||||
embedControls.createEl('span', {
|
||||
text: `${embedInfo.chunk_count} chunks embedded`,
|
||||
cls: 'setting-item-description'
|
||||
});
|
||||
}
|
||||
const buildBtn = embedControls.createEl('button');
|
||||
buildBtn.setText(hasChunks ? t('feat_rebuild_btn') : t('feat_build_btn'));
|
||||
buildBtn.addClass('mod-cta');
|
||||
buildBtn.addEventListener('click', () => startBuild('--resume'));
|
||||
if (!isCorrupted && hasChunks) {
|
||||
const forceBtn2 = embedControls.createEl('button');
|
||||
forceBtn2.setText(t('feat_vector_rebuild_force_btn'));
|
||||
forceBtn2.style.marginLeft = '6px';
|
||||
forceBtn2.addEventListener('click', () => startBuild('--force'));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -3722,8 +3549,7 @@ class PaperForgeSettingTab extends PluginSettingTab {
|
|||
}
|
||||
|
||||
_getCurrentModelKey() {
|
||||
if (this.plugin.settings.vector_db_mode === 'api') return this.plugin.settings.vector_db_api_model || 'openai/text-embedding-3-small';
|
||||
return this.plugin.settings.vector_db_model || 'BAAI/bge-small-en-v1.5';
|
||||
return this.plugin.settings.vector_db_api_model || 'text-embedding-3-small';
|
||||
}
|
||||
|
||||
_parseEmbedStatus(text) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"id": "paperforge",
|
||||
"name": "PaperForge",
|
||||
"version": "1.5.6rc3",
|
||||
"version": "1.5.6rc4",
|
||||
"minAppVersion": "1.9.0",
|
||||
"description": "Zotero literature pipeline for Obsidian. Sync PDFs, run OCR, and read with AI-assisted deep reading.",
|
||||
"author": "Lin Zhaoxuan",
|
||||
|
|
|
|||
|
|
@ -261,6 +261,20 @@ function shouldRenderVectorReady(vectorDepsOk, embedStatusText) {
|
|||
return vectorDepsOk === true;
|
||||
}
|
||||
|
||||
function getDisclosureState(store, key, defaultCollapsed) {
|
||||
if (!store || typeof store !== 'object') return !!defaultCollapsed;
|
||||
if (!Object.prototype.hasOwnProperty.call(store, key)) return !!defaultCollapsed;
|
||||
return !!store[key];
|
||||
}
|
||||
|
||||
function toggleDisclosureState(store, key, defaultCollapsed) {
|
||||
const next = !getDisclosureState(store, key, defaultCollapsed);
|
||||
if (store && typeof store === 'object') {
|
||||
store[key] = next;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
readPathConfig,
|
||||
resolveRuntimePaths,
|
||||
|
|
@ -274,4 +288,6 @@ module.exports = {
|
|||
buildCommandArgs,
|
||||
runSubprocess,
|
||||
shouldRenderVectorReady,
|
||||
getDisclosureState,
|
||||
toggleDisclosureState,
|
||||
};
|
||||
|
|
|
|||
29
paperforge/plugin/tests/settings-panels.test.mjs
Normal file
29
paperforge/plugin/tests/settings-panels.test.mjs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import testable from '../src/testable.js';
|
||||
|
||||
const {
|
||||
getDisclosureState,
|
||||
toggleDisclosureState,
|
||||
} = testable;
|
||||
|
||||
describe('feature panel disclosure state', () => {
|
||||
test('defaults vector config panel to expanded when no state exists', () => {
|
||||
expect(getDisclosureState({}, 'vectorConfig', false)).toBe(false);
|
||||
});
|
||||
|
||||
test('persists collapsed state in the shared panel store', () => {
|
||||
const store = {};
|
||||
|
||||
expect(toggleDisclosureState(store, 'vectorConfig', false)).toBe(true);
|
||||
expect(store.vectorConfig).toBe(true);
|
||||
expect(getDisclosureState(store, 'vectorConfig', false)).toBe(true);
|
||||
});
|
||||
|
||||
test('reopens panel from stored collapsed state', () => {
|
||||
const store = { vectorConfig: true };
|
||||
|
||||
expect(toggleDisclosureState(store, 'vectorConfig', false)).toBe(false);
|
||||
expect(store.vectorConfig).toBe(false);
|
||||
expect(getDisclosureState(store, 'vectorConfig', false)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -40,11 +40,16 @@ $PYTHON -m paperforge --vault "$VAULT" paper-context <zotero_key> --json
|
|||
$PYTHON "$SKILL_DIR/scripts/pf_deep.py" prepare --key <zotero_key> --vault "$VAULT"
|
||||
```
|
||||
|
||||
> **WARNING：`--force` 禁止在精读过程中使用。** `--force` 会清除已有精读内容重新生成骨架，
|
||||
> 仅在用户明确要求重读时才能用。任何时候都不要主动加 `--force`。
|
||||
|
||||
解析返回 JSON：
|
||||
- `status: "ok"` → 记下 `figure_map`、`chart_type_map`、`formal_note`、`fulltext_md`、`figures`、`tables` 的路径和数量
|
||||
- `status: "ok"` → **记下 `figures`（数字）、`tables`（数字）、`figure_map`、`chart_type_map`、`formal_note`、`fulltext_md` 路径**
|
||||
- `status: "warn"` + `deep_reading_status: done` → 告知用户"该文献已精读过"，确认是否重读
|
||||
- `status: "error"` → 报告 `message`，停止
|
||||
|
||||
**把 `figures` 数量记在当前上下文**——Step 4 要用。
|
||||
|
||||
读 formal note，确认 `## 精读` 骨架已插入。
|
||||
|
||||
---
|
||||
|
|
@ -119,6 +124,8 @@ $PYTHON "$SKILL_DIR/scripts/pf_deep.py" prepare --key <zotero_key> --vault "$VAU
|
|||
|
||||
### Step 4: Postprocess（跑校验，修正问题）
|
||||
|
||||
**`N` = Step 1 prepare 输出中的 `figures` 值，不要自己猜。**
|
||||
|
||||
```bash
|
||||
$PYTHON "$SKILL_DIR/scripts/pf_deep.py" postprocess-pass2 "<formal_note_path>" --figures <N>
|
||||
```
|
||||
|
|
@ -152,6 +159,13 @@ $PYTHON "$SKILL_DIR/scripts/pf_deep.py" validate-note "<formal_note_path>" --ful
|
|||
- 输出 `OK` → 告知用户精读完成
|
||||
- 输出错误 → 修正缺失项，直到通过
|
||||
|
||||
**如果 validate 反复失败（超过 3 轮）→ 先自查：**
|
||||
1. 检查 note 结构：`## 🔍 精读` 骨架是否完整？是否被多次 `prepare` 破坏？
|
||||
2. 检查 `Pass 2` 中每个 figure 的子标题是否完整（6 个子标题全部存在？）
|
||||
3. 检查 Pass 1 和 Pass 2 之间是否有内容错位
|
||||
4. 检查 `--figures N` 是否和 paper 实际图片数一致（读 fulltext caption 核实）
|
||||
5. 以上都确认无误还失败 → 报告全部错误给用户，不要继续重试
|
||||
|
||||
---
|
||||
|
||||
## Callout 格式规则
|
||||
|
|
@ -170,3 +184,7 @@ $PYTHON "$SKILL_DIR/scripts/pf_deep.py" validate-note "<formal_note_path>" --ful
|
|||
- 不要在 Pass 1 完成前碰 Pass 2/3
|
||||
- 不要把推断写成文献事实——区分"作者说了 X"和"我推断 Y"
|
||||
- 不要跨 figure 写综合判断（Pass 2 逐图，Pass 3 才做综合）
|
||||
- **禁止主动加 `--force`** — 只有用户明确要求重读时才能用
|
||||
- **禁止猜测 `--figures N`** — N 必须来自 Step 1 prepare 输出的实际数字
|
||||
- **禁止重复跑 `prepare`** — prepare 只跑一次。跑了两次以上 → 检查 note 结构是否被破坏
|
||||
- **validate 失败超过 3 轮 → 执行自查清单，不要盲目重试**
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Discussion recorder -- writes structured AI-paper Q&A into ai/ workspace directory.
|
||||
|
||||
Atomic append-only writes for both JSON (canonical) and Markdown (human-readable).
|
||||
stdlib only -- no dependencies beyond Python standard library.
|
||||
Atomic append-only writes for Markdown (canonical). stdlib only.
|
||||
|
||||
API:
|
||||
record_session(vault_path, zotero_key, agent, model, qa_pairs) -> dict
|
||||
|
|
@ -17,7 +16,6 @@ import argparse
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
|
|
@ -37,8 +35,6 @@ logger = logging.getLogger(__name__)
|
|||
_SCHEMA_VERSION = "1"
|
||||
_ISO_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
|
||||
_MD_SEPARATOR = "---"
|
||||
_MD_SPECIAL_CHARS = re.compile(r"([*#\[\]_`])")
|
||||
"""Regex for markdown special characters that must be escaped in QA text fields."""
|
||||
|
||||
LOCK_TIMEOUT = 10
|
||||
"""Seconds to wait for cross-process file lock before raising filelock.Timeout."""
|
||||
|
|
@ -56,16 +52,6 @@ def _now_iso() -> str:
|
|||
return datetime.now(_CST).isoformat()
|
||||
|
||||
|
||||
def _escape_md(text: str) -> str:
|
||||
"""Escape markdown special characters in QA text fields.
|
||||
|
||||
Escapes: *, #, [, ], _, ` -- prevents broken Obsidian formatting
|
||||
when user questions or AI answers contain these characters.
|
||||
Does not double-escape already-escaped characters.
|
||||
"""
|
||||
return _MD_SPECIAL_CHARS.sub(r"\\\1", text)
|
||||
|
||||
|
||||
def _today_str(iso_stamp: str) -> str:
|
||||
"""Extract YYYY-MM-DD from an ISO 8601 timestamp."""
|
||||
return iso_stamp[:10]
|
||||
|
|
@ -142,35 +128,6 @@ def _build_session(
|
|||
}
|
||||
|
||||
|
||||
def _atomic_write_json(target_path: Path, envelope: dict) -> None:
|
||||
"""Atomically write JSON file via tempfile + os.replace.
|
||||
|
||||
Uses ensure_ascii=False for CJK support, newline='\\n' for consistent
|
||||
line endings on Windows.
|
||||
"""
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = json.dumps(envelope, ensure_ascii=False, indent=2) + "\n"
|
||||
|
||||
fd, tmp_path_str = tempfile.mkstemp(
|
||||
suffix=".json",
|
||||
prefix="discussion_",
|
||||
dir=str(target_path.parent),
|
||||
)
|
||||
tmp_path = Path(tmp_path_str)
|
||||
try:
|
||||
os.write(fd, content.encode("utf-8"))
|
||||
os.close(fd)
|
||||
fd = None
|
||||
os.replace(tmp_path_str, str(target_path))
|
||||
except Exception:
|
||||
# Cleanup temp file on failure
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
if tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
raise
|
||||
|
||||
|
||||
def _build_md_header(paper_title: str) -> str:
|
||||
return f"# AI Discussion Record: {paper_title}\n\n"
|
||||
|
||||
|
|
@ -183,8 +140,8 @@ def _build_md_session(session: dict) -> str:
|
|||
model = session["model"]
|
||||
lines = [f"## {date_str} -- {agent} ({model})\n"]
|
||||
for qa in session["qa_pairs"]:
|
||||
lines.append(f"**问题:** {_escape_md(qa['question'])}")
|
||||
lines.append(f"**解答:** {_escape_md(qa['answer'])}\n")
|
||||
lines.append(f"**问题:** {qa['question']}")
|
||||
lines.append(f"**解答:** {qa['answer']}\n")
|
||||
lines.append(f"{_MD_SEPARATOR}\n")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
|
@ -233,8 +190,7 @@ def record_session(
|
|||
) -> dict:
|
||||
"""Record an AI-paper discussion session.
|
||||
|
||||
Creates or appends to ai/discussion.json (canonical) and
|
||||
ai/discussion.md (human-readable) with atomic writes.
|
||||
Creates or appends to ai/discussion.md (canonical) with atomic writes.
|
||||
|
||||
Args:
|
||||
vault_path: Path to the Obsidian vault root.
|
||||
|
|
@ -284,7 +240,6 @@ def record_session(
|
|||
# Fallback: construct from metadata
|
||||
ai_dir = _build_ai_dir(vault_path, domain, zotero_key, paper_title)
|
||||
|
||||
json_path = ai_dir / "discussion.json"
|
||||
md_path = ai_dir / "discussion.md"
|
||||
|
||||
# --- Build session object ---
|
||||
|
|
@ -293,32 +248,11 @@ def record_session(
|
|||
except Exception as exc:
|
||||
return {"status": "error", "message": f"Error building session: {exc}"}
|
||||
|
||||
# --- Write discussion.json + discussion.md (atomic, locked) ---
|
||||
# HARDEN-01: Cross-process file lock prevents concurrent write corruption.
|
||||
# Uses same filelock pattern as asset_index.py: lock on .json.lock suffix,
|
||||
# timeout after LOCK_TIMEOUT seconds.
|
||||
lock_path = json_path.with_suffix(".json.lock")
|
||||
# --- Write discussion.md (atomic, locked) ---
|
||||
lock_path = md_path.with_suffix(".md.lock")
|
||||
lock = filelock.FileLock(lock_path, timeout=LOCK_TIMEOUT)
|
||||
try:
|
||||
with lock:
|
||||
# Write discussion.json
|
||||
existing_sessions = []
|
||||
if json_path.exists():
|
||||
try:
|
||||
existing_data = json.loads(json_path.read_text(encoding="utf-8"))
|
||||
existing_sessions = existing_data.get("sessions", [])
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
logger.warning("Corrupted discussion.json, starting fresh: %s", exc)
|
||||
existing_sessions = []
|
||||
|
||||
envelope = {
|
||||
"schema_version": "1",
|
||||
"paper_key": zotero_key,
|
||||
"sessions": [*existing_sessions, session],
|
||||
}
|
||||
_atomic_write_json(json_path, envelope)
|
||||
|
||||
# Write discussion.md
|
||||
existing_md = ""
|
||||
if md_path.exists():
|
||||
try:
|
||||
|
|
@ -332,17 +266,13 @@ def record_session(
|
|||
content = _md_content(existing_md, header, session_md)
|
||||
_atomic_write_md(md_path, content)
|
||||
except filelock.Timeout:
|
||||
logger.warning("Could not acquire lock for discussion files: %s", lock_path)
|
||||
logger.warning("Could not acquire lock for discussion file: %s", lock_path)
|
||||
return {"status": "error", "message": "Concurrent access conflict. Please try again."}
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to write discussion files: %s", exc)
|
||||
return {"status": "error", "message": f"Failed to write discussion files: {exc}"}
|
||||
logger.warning("Failed to write discussion file: %s", exc)
|
||||
return {"status": "error", "message": f"Failed to write discussion file: {exc}"}
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"json_path": str(json_path),
|
||||
"md_path": str(md_path),
|
||||
}
|
||||
return {"status": "ok", "md_path": str(md_path)}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -211,6 +211,40 @@ class TestBuildIndexEmpty:
|
|||
envelope = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
assert envelope["paper_count"] == 0
|
||||
|
||||
def test_build_index_does_not_auto_embed_vectors(self, monkeypatch, tmp_path: Path) -> None:
|
||||
"""sync/index build must not trigger vector embedding work."""
|
||||
vault = _minimal_vault(tmp_path)
|
||||
_ensure_domain_config(vault)
|
||||
|
||||
exports_dir = vault / "99_System" / "PaperForge" / "exports"
|
||||
exports_dir.mkdir(parents=True, exist_ok=True)
|
||||
(exports_dir / "library.json").write_text(
|
||||
json.dumps([
|
||||
{
|
||||
"key": "ABC12345",
|
||||
"title": "Paper A",
|
||||
"creators": [],
|
||||
"collections": [],
|
||||
"attachments": [],
|
||||
"doi": "",
|
||||
"pmid": "",
|
||||
"date": "",
|
||||
"extra": "",
|
||||
"abstractNote": "",
|
||||
"publicationTitle": "",
|
||||
"itemType": "journalArticle",
|
||||
}
|
||||
]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _unexpected_auto_embed(_vault, _entry):
|
||||
raise AssertionError("build_index must not auto-embed vectors")
|
||||
|
||||
monkeypatch.setattr("paperforge.worker.asset_index._vec_auto_embed_if_new", _unexpected_auto_embed)
|
||||
|
||||
build_index(vault, verbose=False)
|
||||
|
||||
|
||||
class TestSummarizeIndex:
|
||||
"""summarize_index(vault) -> dict | None"""
|
||||
|
|
|
|||
Loading…
Reference in a new issue