From 4fe55e11dc559af2cfa1f04976edacf9bf70dd0e Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Wed, 6 May 2026 15:14:36 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20restore=20phases=2031-34=20from=20stash?= =?UTF-8?q?=20=E2=80=94=20dashboard=20multi-view,=20deep-reading=20mode,?= =?UTF-8?q?=20jump=20button,=20planning=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Phase 31: Bug fixes (version display, lifecycle alignment, AI Ready removal) - Phase 32: Deep-reading mode detection (_resolveModeForFile, identity guard) - Phase 33: Deep-reading dashboard (status card, Pass 1 extraction, AI Q&A) - Phase 34: Jump to deep reading button (i18n, conditional visibility) - Planning files for phases 31-36 - CSS: Section 33 deep-reading styles - asset_index.py: paperforge_version in envelope --- .planning/PROJECT.md | 40 +- .planning/REQUIREMENTS.md | 102 ++- .planning/ROADMAP.md | 103 ++- .planning/STATE.md | 130 +-- .../01-LEARNINGS.md | 318 +++++++ .../02-LEARNINGS.md | 247 ++++++ .../03-LEARNINGS.md | 175 ++++ .../04-onboarding-validation/04-LEARNINGS.md | 145 ++++ .../05-LEARNINGS.md | 175 ++++ .../06-LEARNINGS.md | 124 +++ .../07-LEARNINGS.md | 124 +++ .../08-deep-helper-deployment/08-LEARNINGS.md | 129 +++ .../09-command-unification/09-LEARNINGS.md | 136 +++ .../10-LEARNINGS.md | 119 +++ .../11-LEARNINGS.md | 172 ++++ .../12-architecture-cleanup/12-LEARNINGS.md | 146 ++++ .../13-logging-foundation/13-LEARNINGS.md | 217 +++++ .../14-00-PLAN.md | 9 + .../14-00-SUMMARY.md | 9 + .../14-CONTEXT.md | 2 +- .../15-LEARNINGS.md | 114 +++ .../16-retry-progress-bars/16-LEARNINGS.md | 156 ++++ .../17-dead-code-precommit/17-LEARNINGS.md | 194 +++++ .../18-LEARNINGS.md | 180 ++++ .planning/phases/19-testing/19-LEARNINGS.md | 154 ++++ .../20-LEARNINGS.md | 154 ++++ .../21-LEARNINGS.md | 226 +++++ .../22-00-PLAN.md} | 0 .../22-00-SUMMARY.md | 9 + .../22-CONTEXT.md | 2 +- .../23-01-PLAN.md | 298 +++++++ .../23-02-PLAN.md | 315 +++++++ .../23-03-PLAN.md | 365 ++++++++ .../24-02-SUMMARY.md | 100 +++ .../25-CONTEXT.md | 109 +++ .../26-01-PLAN.md | 409 +++++++++ .../26-01-SUMMARY.md | 110 +++ .../26-02-PLAN.md | 477 +++++++++++ .../26-03-PLAN.md | 462 ++++++++++ .../phases/27-component-library/27-01-PLAN.md | 698 ++++++++++++++++ .../phases/27-component-library/27-02-PLAN.md | 475 +++++++++++ .../phases/29-per-paper-view/29-01-PLAN.md | 542 ++++++++++++ .planning/phases/31-bug-fixes/31-01-PLAN.md | 69 ++ .planning/phases/31-bug-fixes/31-CONTEXT.md | 39 + .../32-01-PLAN.md | 161 ++++ .../32-CONTEXT.md | 102 +++ .../32-DISCUSSION-LOG.md | 58 ++ .../33-01-PLAN.md | 324 +++++++ .../33-CONTEXT.md | 110 +++ .../33-DISCUSSION-LOG.md | 71 ++ .../34-01-PLAN.md | 242 ++++++ .../34-01-SUMMARY.md | 35 + .../34-VERIFICATION.md | 165 ++++ .planning/research/ARCHITECTURE.md | 787 +++++++----------- .planning/research/FEATURES.md | 532 +++++++++--- .planning/research/MILESTONE-RESEARCH-LOCK.md | 97 +++ .planning/research/PITFALLS.md | 722 ++++++++++++---- .planning/research/STACK.md | 327 ++++---- .planning/research/SUMMARY.md | 415 ++++++--- paperforge/plugin/main.js | 310 +++++-- paperforge/plugin/styles.css | 138 ++- paperforge/worker/asset_index.py | 25 +- tests/sandbox/batch_check.py | 121 +++ tests/sandbox/deep_inspect.py | 42 + tests/sandbox/precise_batch.py | 115 +++ tests/test_ocr_rendering.py | 128 +++ 66 files changed, 12010 insertions(+), 1296 deletions(-) create mode 100644 .planning/phases/01-config-and-command-foundation/01-LEARNINGS.md create mode 100644 .planning/phases/02-paddleocr-and-pdf-path-hardening/02-LEARNINGS.md create mode 100644 .planning/phases/03-obsidian-bases-config-aware/03-LEARNINGS.md create mode 100644 .planning/phases/04-onboarding-validation/04-LEARNINGS.md create mode 100644 .planning/phases/05-workflow-hardening-and-optional-plugin-shell/05-LEARNINGS.md create mode 100644 .planning/phases/06-setup-cli-diagnostics-consistency/06-LEARNINGS.md create mode 100644 .planning/phases/07-zotero-pdf-metadata-state-repair/07-LEARNINGS.md create mode 100644 .planning/phases/08-deep-helper-deployment/08-LEARNINGS.md create mode 100644 .planning/phases/09-command-unification/09-LEARNINGS.md create mode 100644 .planning/phases/10-documentation-and-cohesion/10-LEARNINGS.md create mode 100644 .planning/phases/11-zotero-path-normalization/11-LEARNINGS.md create mode 100644 .planning/phases/12-architecture-cleanup/12-LEARNINGS.md create mode 100644 .planning/phases/13-logging-foundation/13-LEARNINGS.md create mode 100644 .planning/phases/14-shared-utilities-extraction/14-00-PLAN.md create mode 100644 .planning/phases/14-shared-utilities-extraction/14-00-SUMMARY.md create mode 100644 .planning/phases/15-deep-reading-queue-merge/15-LEARNINGS.md create mode 100644 .planning/phases/16-retry-progress-bars/16-LEARNINGS.md create mode 100644 .planning/phases/17-dead-code-precommit/17-LEARNINGS.md create mode 100644 .planning/phases/18-documentation-ux-polish/18-LEARNINGS.md create mode 100644 .planning/phases/19-testing/19-LEARNINGS.md create mode 100644 .planning/phases/20-plugin-settings-shell-persistence/20-LEARNINGS.md create mode 100644 .planning/phases/21-one-click-install-and-polished-ux/21-LEARNINGS.md rename .planning/phases/{22-install-wizard-modal/22-PLAN.md => 22-configuration-truth-compatibility/22-00-PLAN.md} (100%) create mode 100644 .planning/phases/22-configuration-truth-compatibility/22-00-SUMMARY.md create mode 100644 .planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-PLAN.md create mode 100644 .planning/phases/23-canonical-asset-index-safe-rebuilds/23-02-PLAN.md create mode 100644 .planning/phases/23-canonical-asset-index-safe-rebuilds/23-03-PLAN.md create mode 100644 .planning/phases/24-derived-lifecycle-health-maturity/24-02-SUMMARY.md create mode 100644 .planning/phases/25-surface-convergence-doctor-repair/25-CONTEXT.md create mode 100644 .planning/phases/26-traceable-ai-context-packs/26-01-PLAN.md create mode 100644 .planning/phases/26-traceable-ai-context-packs/26-01-SUMMARY.md create mode 100644 .planning/phases/26-traceable-ai-context-packs/26-02-PLAN.md create mode 100644 .planning/phases/26-traceable-ai-context-packs/26-03-PLAN.md create mode 100644 .planning/phases/27-component-library/27-01-PLAN.md create mode 100644 .planning/phases/27-component-library/27-02-PLAN.md create mode 100644 .planning/phases/29-per-paper-view/29-01-PLAN.md create mode 100644 .planning/phases/31-bug-fixes/31-01-PLAN.md create mode 100644 .planning/phases/31-bug-fixes/31-CONTEXT.md create mode 100644 .planning/phases/32-deep-reading-mode-detection/32-01-PLAN.md create mode 100644 .planning/phases/32-deep-reading-mode-detection/32-CONTEXT.md create mode 100644 .planning/phases/32-deep-reading-mode-detection/32-DISCUSSION-LOG.md create mode 100644 .planning/phases/33-deep-reading-dashboard-rendering/33-01-PLAN.md create mode 100644 .planning/phases/33-deep-reading-dashboard-rendering/33-CONTEXT.md create mode 100644 .planning/phases/33-deep-reading-dashboard-rendering/33-DISCUSSION-LOG.md create mode 100644 .planning/phases/34-jump-to-deep-reading-button/34-01-PLAN.md create mode 100644 .planning/phases/34-jump-to-deep-reading-button/34-01-SUMMARY.md create mode 100644 .planning/phases/34-jump-to-deep-reading-button/34-VERIFICATION.md create mode 100644 .planning/research/MILESTONE-RESEARCH-LOCK.md create mode 100644 tests/sandbox/batch_check.py create mode 100644 tests/sandbox/deep_inspect.py create mode 100644 tests/sandbox/precise_batch.py create mode 100644 tests/test_ocr_rendering.py diff --git a/.planning/PROJECT.md b/.planning/PROJECT.md index a82e4b9a..817d9032 100644 --- a/.planning/PROJECT.md +++ b/.planning/PROJECT.md @@ -2,18 +2,28 @@ ## What This Is -PaperForge is a local-first Obsidian + Zotero literature asset manager. It turns Zotero exports, PDFs, OCR fulltext, figures, notes, and AI outputs into a structured, traceable, reusable research library. v1.6 adds unified configuration, canonical asset indexing, derived lifecycle/health/maturity, surface convergence, AI context entry points, and safe workspace migration. +PaperForge is a local-first Obsidian + Zotero literature asset manager. It turns Zotero exports, PDFs, OCR fulltext, figures, notes, and AI outputs into a structured, traceable, reusable research library. v1.7 adds context-aware dashboard routing with lifecycle/health/maturity visualization. v1.8 adds AI discussion recording and deep-reading dashboard integration. -## Current Milestone: v1.7 Context-Aware Dashboard +## Current Milestone: v1.8 AI Discussion & Deep-Reading Dashboard -**Goal:** Make PaperForge's plugin dashboard context-aware—showing different views based on the active file (Base, paper card, or global). Uses pure CSS/DOM components: metric cards, lifecycle stepper, health matrix, maturity gauge, bar charts. +**Goal:** Capture AI-paper discussions into structured ai/ records and extend the per-paper dashboard with deep-reading content and AI interaction history. **Target features:** -- Auto-detect active file type and switch dashboard mode accordingly. -- Per-paper dashboard with lifecycle stepper, health matrix, maturity gauge, and next-step guidance. -- Collection dashboard for Base files with domain-level lifecycle/health aggregation. -- Global dashboard retains and enhances existing library overview. -- Pure CSS component library (no npm dependencies). +- AI discussion recorder producing `discussion.md` (human-readable Q&A) + `discussion.json` (dashboard-consumable) in each paper's workspace `ai/` directory. +- Deep-reading dashboard mode: status bar, Pass 1 full-text summary, and AI Q&A history when viewing `deep-reading.md`. +- "Jump to Deep Reading" button on the per-paper dashboard card. +- Bug fixes: remove meaningless "ai" UI row; restore version number display. + +## Completed Milestone: v1.7 Context-Aware Dashboard + +**Status:** COMPLETE (2026-05-04) + +**Delivered:** +- Pure CSS dashboard components (metric cards, lifecycle stepper, health matrix, maturity gauge, bar chart) +- Mode-aware dashboard routing with auto-detect and debounced switching +- Full per-paper dashboard: lifecycle stepper, health matrix, maturity gauge, next-step recommendations +- Collection dashboard: domain-level metric cards, lifecycle bar chart, health grid aggregation +- Global dashboard with enhanced library overview ## Completed Milestone: v1.6 AI-Ready Literature Asset Foundation @@ -115,13 +125,11 @@ Researchers always know what papers they have, what state those papers are in, a ### Active -- [ ] Single configuration truth across plugin, CLI, worker, and setup flows. -- [ ] Canonical literature asset index derived from existing sync/OCR/deep-reading outputs. -- [ ] Stable lifecycle model for imported, indexed, OCR-ready, fulltext-ready, figure-ready, deep-read, and AI-context-ready states. -- [ ] Health diagnostics that expose broken PDFs, OCR failures, path drift, template/Base drift, and incomplete assets. -- [ ] Plugin Dashboard that reads the canonical index instead of recomputing state independently. -- [ ] Library Maturity scoring and next-step recommendations. -- [ ] Reusable AI context packaging without hardcoding discipline-specific extraction schemas. +- [ ] AI discussion recorder: `/pf-paper` and agent chats produce `discussion.md` (Q&A) + `discussion.json` (structured) in workspace `ai/`. +- [ ] Deep-reading dashboard mode with status bar, Pass 1 summary, and AI Q&A history. +- [ ] "Jump to Deep Reading" button on per-paper dashboard card. +- [ ] Bug fix: remove meaningless "ai" row from plugin UI. +- [ ] Bug fix: restore version number display in plugin. ### Out of Scope @@ -239,4 +247,4 @@ This document evolves at phase transitions and milestone boundaries. 4. Update Context with current state. --- -*Last updated: 2026-05-03 after milestone v1.6 initialization* +*Last updated: 2026-05-06 after milestone v1.8 initialization* diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 2721891f..944ab77c 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -1,86 +1,78 @@ # Requirements: PaperForge -**Defined:** 2026-05-04 +**Defined:** 2026-05-06 **Core Value:** Researchers always know what papers they have, what state those papers are in, and whether each paper is reliably usable by AI with traceable fulltext, figures, notes, and source links. -## v1 Requirements +## v1.8 Requirements -Requirements for milestone v1.7: Context-Aware Dashboard. +### AI Discussion Recording -### Context Detection +- [ ] **AI-01**: `paperforge/worker/discussion.py` writes discussion.md (human-readable Q&A, `问题:` / `解答:` format, chronological sections) into paper workspace `ai/` directory. +- [ ] **AI-02**: `paperforge/worker/discussion.py` writes discussion.json (structured, sessions[] array with `schema_version`, `timestamp`, `qa_pairs[]` per session) into paper workspace `ai/` directory. +- [ ] **AI-03**: `/pf-paper` and `/pf-deep` agent sessions trigger discussion recorder at session completion, producing both files. -- [x] **DASH-01**: User opens a `.base` file — plugin dashboard shows collection-level domain statistics. -- [x] **DASH-02**: User opens a paper card (`.md` with `zotero_key` in frontmatter) — plugin dashboard shows per-paper lifecycle, health, maturity, and next-step. -- [x] **DASH-03**: User opens any other file or no file — plugin dashboard shows the existing global library overview. -- [x] **DASH-04**: User switches active file — dashboard auto-refreshes to the correct mode without manual intervention. +### Deep-Reading Dashboard -### Per-Paper Dashboard +- [ ] **DEEP-01**: `_detectAndSwitch()` recognizes `deep-reading.md` as the `deep-reading` mode, checked BEFORE the `zotero_key` branch to prevent routing to per-paper mode. +- [ ] **DEEP-02**: `_renderDeepReadingMode()` renders: status bar (figure-map, OCR state, Pass 1/2/3 completion), Pass 1 full-text summary card (extracted from deep-reading.md), and AI Q&A history card (from `discussion.json`). +- [ ] **DEEP-03**: All four empty-state conditions render user-facing messages ("暂无" or equivalent) rather than errors: (a) missing discussion.json, (b) empty sessions, (c) missing Pass 1 content, (d) deep-reading.md not found. -- [x] **PAPER-01**: User sees a lifecycle stepper showing the current state and which stages are complete. -- [x] **PAPER-02**: User sees a health matrix (PDF/OCR/Note/Asset dimensions) with color-coded status. -- [x] **PAPER-03**: User sees maturity level (1-6) as a segmented progress bar with blocking checks listed. -- [x] **PAPER-04**: User sees a recommended next step (sync/ocr/pf-deep/ready) with an action trigger. +### Navigation & Polish -### Collection Dashboard +- [ ] **NAV-01**: Per-paper dashboard card has a "Jump to Deep Reading" button that opens `deep-reading.md` via `openLinkText()`, verifying file existence with `getAbstractFileByPath()` before navigating. +- [ ] **NAV-02**: Plugin version number is read and displayed (from `paperforge/__init__.py` or `manifest.json` build-time embedding). +- [ ] **NAV-03**: Meaningless "ai" row is removed from plugin rendering. -- [x] **COLL-01**: User sees domain-level paper count and lifecycle distribution from canonical index. -- [x] **COLL-02**: User sees aggregated health overview for the domain (PDF/OCR/Note/Asset counts). -- [x] **COLL-03**: User sees a lifecycle distribution bar chart for the domain. +### Integration Verification -### Component Library - -- [x] **COMP-01**: All dashboard visualizations use pure CSS/DOM (metric cards, lifecycle stepper, health matrix, maturity gauge, bar charts). No npm dependencies. -- [x] **COMP-02**: Components use Obsidian CSS variables for consistent theming. -- [x] **COMP-03**: Components have loading states, CSS transitions, and responsive breakpoints. - -### Auto-Refresh - -- [x] **REFR-01**: Dashboard refreshes when the canonical index file changes. -- [x] **REFR-02**: Dashboard refreshes when the active file changes. +- [ ] **INTEG-01**: End-to-end test: `/pf-paper` → discussion files created → dashboard shows AI Q&A history on deep-reading.md open. Windows CJK encoding verified; no `btoa()` on Chinese paths. ## v2 Requirements -Deferred to future milestone. +Deferred to future release. Tracked but not in current roadmap. -### LLMWiki +### AI Discussion Recording -- **LLM-01**: User can explore a cross-paper concept network built from AI atoms and canonical index entries. -- **LLM-02**: User can navigate concept pages with source traceability back to originating papers. +- **AI-04**: Threaded discussion format (nested Q&A threads within sessions, instead of flat qa_pairs[]). +- **AI-05**: Discussion search/retrieval across all papers by keyword or date range. + +### Deep-Reading Dashboard + +- **DEEP-04**: Inline editing of Pass 1 summary from dashboard (two-way sync with deep-reading.md). +- **DEEP-05**: Figure thumbnail previews in dashboard status bar (embeds from figure-map.json images). ## Out of Scope | Feature | Reason | |---------|--------| -| LLMWiki concept network | v1.8 — depends on dashboard being stable first | -| External chart libraries (Chart.js, D3) | Pure CSS/DOM keeps plugin self-contained | -| Plugin auto-update | Deferred to Obsidian Community Plugins listing | +| Auto-recording every agent interaction | Recording is voluntary/session-end only — agents must explicitly trigger. Auto-capture would produce noise and violate user privacy expectations. | +| Dashboard replacing deep-reading.md | Dashboard is an index INTO the deep-reading note, not a replacement. Full content lives in .md file. | +| Real-time dashboard syncing during agent sessions | Dashboard refreshes on active-leaf-change only. Polling or websocket sync would violate thin-shell constraint. | +| Discussion analytics/categorization | V1.8 records raw Q&A. Tagging, categorization, and cross-paper analysis deferred to v2. | +| Figure preview carousel in dashboard | Deferred to v2. Status bar shows figure count and map existence only. | ## Traceability +Which phases cover which requirements. Updated during roadmap creation. + | Requirement | Phase | Status | |-------------|-------|--------| -| COMP-01 | Phase 27 | Complete | -| COMP-02 | Phase 27 | Complete | -| COMP-03 | Phase 27 | Complete | -| DASH-01 | Phase 28 | Complete | -| DASH-02 | Phase 28 | Complete | -| DASH-03 | Phase 28 | Complete | -| DASH-04 | Phase 28 | Complete | -| REFR-01 | Phase 28 | Complete | -| REFR-02 | Phase 28 | Complete | -| PAPER-01 | Phase 29 | Complete | -| PAPER-02 | Phase 29 | Complete | -| PAPER-03 | Phase 29 | Complete | -| PAPER-04 | Phase 29 | Complete | -| COLL-01 | Phase 30 | Complete | -| COLL-02 | Phase 30 | Complete | -| COLL-03 | Phase 30 | Complete | +| AI-01 | Phase 35 | Pending | +| AI-02 | Phase 35 | Pending | +| AI-03 | Phase 35 | Pending | +| DEEP-01 | Phase 32 | Pending | +| DEEP-02 | Phase 33 | Pending | +| DEEP-03 | Phase 33 | Pending | +| NAV-01 | Phase 34 | Pending | +| NAV-02 | Phase 31 | Pending | +| NAV-03 | Phase 31 | Pending | +| INTEG-01 | Phase 36 | Pending | **Coverage:** -- v1 requirements: 16 total -- Completed: 16 -- Pending: 0 ✓ +- v1.8 requirements: 10 total +- Mapped to phases: 10 ✓ +- Unmapped: 0 --- -*Requirements defined: 2026-05-04* -*Last updated: 2026-05-04 after v1.7 requirements definition* +*Requirements defined: 2026-05-06* +*Last updated: 2026-05-06 — traceability updated during roadmap creation* diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index d9cf94bb..f1408076 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -1,7 +1,7 @@ # Roadmap: PaperForge -**Current milestone:** v1.8 — Planned -**Phase numbering:** Continuous. v1.7 ended at Phase 30. +**Current milestone:** v1.8 AI Discussion & Deep-Reading Dashboard — Planned +**Phase numbering:** Continuous. v1.7 ended at Phase 30. v1.8 begins at Phase 31. --- @@ -15,6 +15,7 @@ - ✅ **v1.5 Obsidian Plugin Setup Integration** — Phases 20-21 (shipped 2026-04-29) - ✅ **v1.6 AI-Ready Literature Asset Foundation** — Phases 22-26 (shipped 2026-05-04) - ✅ **v1.7 Context-Aware Dashboard** — Phases 27-30 (shipped 2026-05-04) +- 📋 **v1.8 AI Discussion & Deep-Reading Dashboard** — Phases 31-36 (planned) *Archive: `.planning/milestones/`* @@ -43,6 +44,96 @@ +### 📋 v1.8 AI Discussion & Deep-Reading Dashboard (Planned) + +**Milestone Goal:** Capture AI-paper discussions into structured ai/ records and extend the per-paper dashboard with deep-reading content and AI interaction history. + +- [ ] **Phase 31: Bug Fixes** — Restore version display; remove meaningless "ai" UI row +- [ ] **Phase 32: Deep-Reading Mode Detection** — Plugin routes deep-reading.md to dedicated dashboard mode +- [ ] **Phase 33: Deep-Reading Dashboard Rendering** — Status bar, Pass 1 summary, empty-state AI Q&A card +- [x] **Phase 34: Jump to Deep Reading Button** — Per-paper dashboard card links to deep-reading.md (completed 2026-05-06) +- [ ] **Phase 35: AI Discussion Recorder** — Python module writes discussion.md + discussion.json into ai/ +- [ ] **Phase 36: Integration Verification** — End-to-end pipeline verified with CJK encoding and vault.adapter.read + +--- + +## Phase Details + +### Phase 31: Bug Fixes +**Goal**: Plugin version number is displayed correctly and the meaningless "ai" row is removed from all dashboard views. +**Depends on**: Nothing (first phase) +**Requirements**: NAV-02, NAV-03 +**Success Criteria** (what must be TRUE): + 1. Plugin header shows the actual PaperForge version (e.g., "v1.8.0") read from package metadata, not a placeholder like "v—" + 2. Dashboard rendering excludes the empty "ai" row across all three render modes (global, paper, collection) + 3. git grep confirms zero references to the removed "ai" row rendering logic in plugin JS and CSS + 4. The Python→JS version bridge (paperforge_version in formal-library.json envelope) is established for future consumption +**Plans**: TBD +**UI hint**: yes + +### Phase 32: Deep-Reading Mode Detection +**Goal**: Plugin detects deep-reading.md files by filename (before zotero_key frontmatter check) and routes to a dedicated `deep-reading` dashboard mode without oscillation. +**Depends on**: Phase 31 +**Requirements**: DEEP-01 +**Success Criteria** (what must be TRUE): + 1. Opening any deep-reading.md file in the vault switches the dashboard to deep-reading mode, validating the parent directory pattern ({8-char key} - {title}) + 2. A deep-reading.md file carrying zotero_key frontmatter routes to deep-reading mode, NOT per-paper mode — filename check precedes frontmatter check + 3. Switching away from deep-reading.md (e.g., opening a formal note) transitions to the correct mode without visible flicker or mode oscillation + 4. `_resolveModeForFile()` is extracted as a pure function with identity guard (same mode AND same file path = no-op) to prevent active-leaf-change double-fire +**Plans**: TBD +**UI hint**: yes + +### Phase 33: Deep-Reading Dashboard Rendering +**Goal**: Deep-reading dashboard renders reading progress status, Pass 1 full-text summary, and graceful empty states for all four AI discussion data conditions. +**Depends on**: Phase 32 +**Requirements**: DEEP-02, DEEP-03 +**Success Criteria** (what must be TRUE): + 1. Deep-reading dashboard displays a status bar with Pass 1/2/3 completion indicators and OCR/health/maturity badges + 2. Pass 1 full-text summary card renders content extracted from deep-reading.md markers (e.g., **一句话总览**), supporting regex fallback for formatting variations + 3. AI Q&A history section shows a descriptive Chinese placeholder ("暂无讨论记录") when no discussion.json exists, and "暂无问答内容" when sessions are empty + 4. All four empty-state conditions render user-facing messages without JavaScript errors: (a) missing discussion.json, (b) empty sessions array, (c) missing Pass 1 content, (d) deep-reading.md file not found + 5. CSS additions are scoped under `.paperforge-mode-deepreading` wrapper class with `paperforge-deepreading-*` component prefixes to avoid namespace collisions +**Plans**: TBD +**UI hint**: yes + +### Phase 34: Jump to Deep Reading Button +**Goal**: Per-paper dashboard card provides a one-click contextual button to open the associated deep-reading.md file, with file-existence verification before navigation. +**Depends on**: Phase 32, Phase 33 +**Requirements**: NAV-01 +**Success Criteria** (what must be TRUE): + 1. Per-paper dashboard card renders a "跳转到精读" button when the paper's deep_reading_path exists and deep_reading_status is 'done' + 2. Clicking the button opens deep-reading.md via openLinkText() in the active leaf, which triggers Phase 32 mode detection and renders the Phase 33 dashboard + 3. The button is hidden (not just disabled) when the paper has no deep_reading_path or its status is not 'done' + 4. If deep-reading.md is missing from disk despite the index claiming it exists, app.vault.adapter.getAbstractFileByPath() returns null and a clear Obsidian Notice informs the user instead of crashing +**Plans**: 1 plan +**UI hint**: yes + +### Phase 35: AI Discussion Recorder +**Goal**: Agent sessions (`/pf-paper`, `/pf-deep`) produce structured discussion records — JSON (canonical) and Markdown (human-readable) — in each paper's workspace `ai/` directory with atomic writes and UTF-8 encoding. +**Depends on**: Nothing (standalone Python module; can execute in parallel with Phases 32-34) +**Requirements**: AI-01, AI-02, AI-03 +**Success Criteria** (what must be TRUE): + 1. Running `/pf-paper` or `/pf-deep` for a paper creates `ai/discussion.json` in that paper's workspace directory, containing `schema_version: "1"`, `paper_key`, and a `sessions[]` array with `session_id`, `agent`, `started`, `model`, and `qa_pairs[]` (each with `question`, `answer`, `source`, `timestamp`) + 2. The same session produces `ai/discussion.md` with human-readable `问题:` / `解答:` format, chronological `##` session headings, and timestamp metadata + 3. Re-running an agent session for the same paper appends a new session entry rather than overwriting previous discussions, using atomic read-modify-write via `tempfile.NamedTemporaryFile` + `os.replace()` + 4. Discussion files are written with explicit `encoding='utf-8'` and `newline='\n'` and are readable in Obsidian without mojibake on Windows CJK systems + 5. `discussion.py` uses only Python stdlib (`json`, `pathlib`, `datetime`, `tempfile`, `os`) — zero new dependencies +**Plans**: TBD + +### Phase 36: Integration Verification +**Goal**: End-to-end pipeline verified: agent session writes discussion files → dashboard reads and renders AI Q&A history via vault.adapter.read(). Windows CJK encoding and no-btoa() constraints validated. +**Depends on**: Phase 33, Phase 35 +**Requirements**: INTEG-01 +**Success Criteria** (what must be TRUE): + 1. Full flow works end-to-end: run `/pf-paper` → verify discussion.json and discussion.md exist in `ai/` → open deep-reading.md → dashboard AI Q&A History section renders Q&A pairs from discussion.json (last 3 pairs across all sessions, most recent first) + 2. Chinese content (questions, answers, paper titles) survives the Python→disk→JS pipeline without encoding corruption — verified on Windows with CJK locale + 3. Zero `btoa()` or `atob()` calls exist in any path construction involving paper titles or discussion file paths — uses `Buffer.from(str, 'utf-8').toString('base64')` if base64 needed + 4. Dashboard discussion.json read uses `app.vault.adapter.read()`, not `fs.readFileSync()`, and the vault modify event listener includes `discussion.json` paths to trigger dashboard refresh when Python appends new Q&A +**Plans**: TBD +**UI hint**: yes + +--- + ## Progress | Phase | Plans | Status | Completed | @@ -51,7 +142,13 @@ | 28. Dashboard Shell & Context Detection | 2/2 | Complete | 2026-05-04 | | 29. Per-Paper View | 1/1 | Complete | 2026-05-04 | | 30. Collection View | 1/1 | Complete | 2026-05-04 | +| 31. Bug Fixes | 0/TBD | Not started | - | +| 32. Deep-Reading Mode Detection | 0/TBD | Not started | - | +| 33. Deep-Reading Dashboard Rendering | 0/TBD | Not started | - | +| 34. Jump to Deep Reading Button | 1/1 | Complete | 2026-05-06 | +| 35. AI Discussion Recorder | 0/TBD | Not started | - | +| 36. Integration Verification | 0/TBD | Not started | - | --- -*Roadmap updated: 2026-05-04 — v1.7 milestone completed* +*Roadmap updated: 2026-05-06 — v1.8 milestone planned* diff --git a/.planning/STATE.md b/.planning/STATE.md index 5b8d7bf1..59d5f89d 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -1,73 +1,37 @@ --- gsd_state_version: 1.0 -milestone: v1.7 -milestone_name: Context-Aware Dashboard -status: v1.7 milestone complete -stopped_at: Completed 30-01-PLAN.md -last_updated: "2026-05-04T14:22:05.370Z" +milestone: v1.8 +milestone_name: AI Discussion & Deep-Reading Dashboard +status: Ready to plan +stopped_at: Phase 35 context gathered +last_updated: "2026-05-06T06:55:03.559Z" progress: - total_phases: 4 - completed_phases: 4 - total_plans: 6 - completed_plans: 6 + total_phases: 6 + completed_phases: 1 + total_plans: 4 + completed_plans: 1 --- # Project State ## Project Reference -See: .planning/PROJECT.md (updated 2026-05-03) +See: .planning/PROJECT.md (updated 2026-05-06) -**Core value:** Researchers always know what papers they have, what state those papers are in, and whether each paper is reliably usable by AI with traceable fulltext, figures, notes, and source links. -**Current focus:** Phase 23 — canonical-asset-index-safe-rebuilds +**Core value:** Researchers always know what papers have, what state those papers are in, and whether each paper is reliably usable by AI with traceable fulltext, figures, notes, and source links. +**Current focus:** Phase 34 — Jump to Deep Reading Button ## Current Position -Phase: 30 (collection-view) — COMPLETE -Plans: 1 of 1 planned - -Milestone v1.7 (Context-Aware Dashboard): ALL PHASES COMPLETE +Phase: 35 +Plan: Not started ## Performance Metrics **Velocity:** -- Total plans completed: 41 +- Total plans completed: 42 (across v1.0-v1.8) - Average duration: Not yet tracked consistently -- Total execution time: Not yet tracked consistently - -**By Phase:** - -| Phase | Plans | Total | Avg/Plan | -|-------|-------|-------|----------| -| 20. Plugin Settings Shell & Persistence | 1/1 | Not tracked | Not tracked | -| 21. One-Click Install & Polished UX | 2/2 | Not tracked | Not tracked | -| 22-26. v1.6 roadmap | 0/TBD | - | - | - -**Recent Trend:** - -- Last 5 plans: Not normalized in historical records -- Trend: Stable - -| Phase 22-configuration-truth-compatibility P01 | 4 min | 3 tasks | 3 files | -| Phase 22-configuration-truth-compatibility P02 | 6 min | 3 tasks | 1 files | -| Phase 22-configuration-truth-compatibility P03 | 8 min | 3 tasks | 3 files | -| Phase 23-canonical-asset-index-safe-rebuilds P01 | 7 min | 3 tasks | 3 files | -| Phase 23-canonical-asset-index-safe-rebuilds P02 | 5 min | 3 tasks | 4 files | -| Phase 23-canonical-asset-index-safe-rebuilds P03 | 8 min | 4 tasks | 5 files | -| Phase 24-derived-lifecycle-health-maturity P02 | 25min | 3 tasks | 2 files | -| Phase 25-surface-convergence-doctor-repair P02 | 2min | 2 tasks | 1 files | -| Phase 25-surface-convergence-doctor-repair P01 | 5 min | 3 tasks | 4 files | -| Phase 25-surface-convergence-doctor-repair P03 | 13min | 2 tasks | 5 files | -| Phase 26-traceable-ai-context-packs P02 | 60 min | 3 tasks | 5 files | -| Phase 26-traceable-ai-context-packs P01 | 4 min | 2 tasks | 3 files | -| Phase 26-traceable-ai-context-packs P03 | 2 min | 3 tasks | 1 files | -| Phase 27-component-library P01 | 3 min | 3 tasks | 1 files | -| Phase 27-component-library P02 | 3 min | 3 tasks | 1 files | -| Phase 28-dashboard-shell-context-detection P01 | 1 min | 2 tasks | 2 files | -| Phase 28-dashboard-shell-context-detection P02 | 5 min | 2 tasks | 1 files | -| Phase 29 P01 | 10 min | 2 tasks | 2 files | -| Phase 30 P01 | 2 min | 2 tasks | 2 files | ## Accumulated Context @@ -76,56 +40,34 @@ Milestone v1.7 (Context-Aware Dashboard): ALL PHASES COMPLETE Decisions are logged in PROJECT.md Key Decisions table. Recent decisions affecting current work: -- v1.6 stays Python-first: config, lifecycle, health, maturity, and context-pack rules remain Python-owned. -- `formal-library.json` evolves into the canonical derived asset index rather than introducing a parallel index. -- Plugin remains a thin shell over CLI logic and canonical index outputs. -- [Phase 22-configuration-truth-compatibility]: schema_version is metadata excluded from load_vault_config() path config output; use get_paperforge_schema_version() instead -- [Phase 22-configuration-truth-compatibility]: Added paddleocr_api_key and zotero_data_dir to DEFAULT_SETTINGS to prevent data loss from saveSettings() key filtering — Plan omitted these keys from DEFAULT_SETTINGS, but saveSettings() now filters persisted keys to only DEFAULT_SETTINGS entries - would have permanently deleted user API keys and Zotero paths -- [Phase 22-configuration-truth-compatibility]: Clean dict replace replaces existing_config.update() to avoid accumulating stale top-level keys in setup wizard paperforge.json output -- [Phase 23-canonical-asset-index-safe-rebuilds]: Lazy imports inside build_index avoid circular import between sync.py and asset_index.py -- [Phase 23-canonical-asset-index-safe-rebuilds]: Orphaned-record cleanup stays in sync.py; only the core build loop moves to asset_index -- [Phase 23-canonical-asset-index-safe-rebuilds]: OCR captures done keys before queue filter to pass to incremental refresh; deep-reading refreshes ALL records; repair triggers refresh for path and divergence fixes; run_index_refresh keeps full-rebuild default; incremental is opt-in from single-paper workers -- [Phase 24-derived-lifecycle-health-maturity]: Lazy import of asset_state functions inside _build_entry() follows existing pattern — avoids circular dependency risk with sync.py — Same pattern already used for ocr.py and sync.py imports in _build_entry() -- [Phase 24-derived-lifecycle-health-maturity]: Derived fields inserted after entry dict construction but before formal note write — keeps machine-only fields out of user-visible Obsidian note frontmatter — Lifecycle/health/maturity/next_step are machine-derived and should not appear in user-editable markdown frontmatter -- [Phase 25-surface-convergence-doctor-repair]: Plugin dashboard reads formal-library.json directly via readFileSync instead of spawning Python CLI — SURF-03 per D-05, D-06, D-07: plugin consumes same canonical semantics from canonical index -- [Phase 25-surface-convergence-doctor-repair]: Use English column display names (Lifecycle, Maturity, Next Step) in Base view properties — The agent's discretion — Base is a technical view, English labels suffice -- [Phase 25-surface-convergence-doctor-repair]: Double-quote YAML wrapping for filters containing single-quoted lifecycle values — Prevents YAML parse errors when filter values contain single quotes like lifecycle = 'fulltext_ready' -- [Phase 25-surface-convergence-doctor-repair]: Lazy import build_index inside fix conditional block to avoid circular dependency with asset_index — Follows existing lazy import pattern established in Phase 23 -- [Phase 26-traceable-ai-context-packs]: context command wraps canonical index entries with _provenance (9 path keys) and _ai_readiness (blocking explanation) — D-01, D-06, D-09, D-10 -- [Phase 26-traceable-ai-context-packs]: Migration runs before build_index() — ensures _build_entry sees workspace dir -- [Phase 26-traceable-ai-context-packs]: Collection context defaults to --all filter (no Base view filter reading yet in initial implementation) -- [Phase 26-traceable-ai-context-packs]: Context actions use variable timeout: 30s for single paper, 60s for collection, 600s for existing sync/ocr/doctor/repair -- [Phase 27-component-library]: Gauge gradient progression uses cyan/blue/purple/green/yellow/red per-level colors matching lifecycle stage mapping — Visual consistency across gauge and bar chart components at agent discretion per D-17-D-18 -- [Phase 27-component-library]: Pure CSS tooltip via [title]:hover::after/::before with arrow pointer using Obsidian CSS variables — No JS required for tooltips per D-16; keeps component CSS-only -- [Phase 27-component-library]: Status classes applied via variable in _renderHealthMatrix -- enables DRY handling of healthy/warning/failed status values -- [Phase 27-component-library]: Bar fill CSS classes use template literal for dynamic stage color -- matches Plan 27-01 color variant selectors -- [Phase 27-component-library]: Bar chart returns empty state rather than skeleton when data is empty -- more informative for user -- [Phase 28-dashboard-shell-context-detection]: _loadIndex() returns null (not empty object) on failure so callers can distinguish missing/corrupt from empty index (D-17) -- [Phase 28-dashboard-shell-context-detection]: _getCachedIndex() returns [] when index missing, not null — callers iterate safely without null checks (D-14) -- [Phase 28-dashboard-shell-context-detection]: Path resolution in _loadIndex() duplicates _fetchStats() intentionally — self-contained, no hidden coupling; existing _fetchStats() left untouched -- [Phase 28-dashboard-shell-context-detection 02]: 300ms debounce for active-leaf-change based on agent discretion in CONTEXT.md -- [Phase 28-dashboard-shell-context-detection 02]: OCR components created in _renderGlobalMode() not _buildPanel() for lean structural shell -- [Phase 28-dashboard-shell-context-detection 02]: Event refs stored as {event, ref} objects for Obsidian version compatibility +- v1.8 roadmap: 6 phases (31-36) covering 10 requirements across bug fixes, mode detection, dashboard rendering, navigation, AI recording, and integration. +- discussion.json uses sessions-based schema (sessions[] → qa_pairs[]) with schema_version "1" envelope from day one. +- Plugin reads discussion.json via app.vault.adapter.read() (NOT fs.readFileSync) because it lives in vault-internal paper workspace. +- Mode detection checks deep-reading.md filename BEFORE zotero_key frontmatter to prevent per-paper mode hijacking. +- Phase 35 (Python) runs parallel to Phases 32-34 (JS) — no runtime dependency. +- [Phase 31] paperforge_version added to formal-library.json envelope for version display in plugin header. +- [Phase 31] "AI Ready" lifecycle stage removed from plugin dashboard (unreachable key mismatch: Python returns ai_context_ready, JS had ai_ready). +- [Phase 31] Lifecycle stage keys in plugin aligned with Python compute values: deep_read_done not deep_read. +- [Phase 31] Collection mode lifecycle thresholds fixed to match actual lifecycle values instead of mismatched keys. +- [Phase 33] _renderDeepReadingMode() is async — reads deep-reading.md and discussion.json via vault.read(). Contains modeGuard for race condition safety. +- [Phase 33] Pass 1 extraction uses marker priority: **一句话总览** → ## Pass 1 → **文章摘要**, cuts at next major section break. +- [Phase 33] AI Q&A: sessions-based collapsible groups, dialog bubble format (question/answer different colors), default collapsed. ### Pending Todos None yet. -- [Phase 29-per-paper-view]: _renderPaperMode() uses entry.next_step string → maps to 6-state stepInfo object for human-readable label + action trigger, with /pf-deep as clipboard copy (no CLI action) -- [Phase 29-per-paper-view]: Contextual action row created inside _renderPaperMode() before components — Copy Context reuses existing ACTIONS.paperforge-copy-context (needsKey), Open Fulltext uses new _openFulltext() with vault.getAbstractFileByPath + workspace.openLinkText -- [Phase 29-per-paper-view]: Next-step action trigger for sync/ocr/repair calls ACTIONS.find by cmd + _runAction(); /pf-deep copies key via clipboard; ready shows "Copy Context" shortcut — all three paths are explicit in _renderNextStepCard -- [Phase 30-collection-view]: Fulltext-ready threshold = lifecycle in [fulltext_ready, deep_read, ai_ready]; deep-read threshold = lifecycle in [deep_read, ai_ready] -- [Phase 30-collection-view]: Health dimension labels: PDF (Healthy/Broken), OCR (Done/Pending-Failed), Note (Present/Missing), Asset (Valid/Drifted) -- [Phase 30-collection-view]: Metric cards: Papers (cyan, no bar), Fulltext Ready (green, progress bar), Deep Read (yellow, progress bar) -- [Phase 30-collection-view]: Health aggregation computed inline from domain-filtered items (not from _cachedStats global aggregate) - ### Blockers/Concerns -- Brownfield rollout must protect existing vaults, old Base templates, partial OCR assets, and legacy config shapes. -- AI context entry points should ship only after provenance and readiness are trustworthy. +- Agent integration surface for discussion recording: exact callback/handoff protocol for `/pf-paper` and `/pf-deep` completion needs implementation-level confirmation during Phase 35 planning. +- [Phase 33 resolved] Deep-reading.md content parsing: marker-based extraction with regex fallback now implemented. Content includes `**一句话总览**` paragraph + `###` sub-sections. +- [Phase 32 resolved] active-leaf-change double-fire — mitigated by `_currentMode + _currentFilePath` identity guard in `_switchMode()`. +- [Phase 32 resolved] Mode detection — `_resolveModeForFile()` pure function checks deep-reading.md filename + parent directory pattern BEFORE zotero_key frontmatter. +- [Phase 31 resolved] Version display bug fixed — paperforge_version now flows through formal-library.json envelope. Version shown matches __init__.py version (currently 1.4.15). +- [Phase 31 resolved] "AI Ready" row removed from per-paper dashboard lifecycle stepper and bar chart. ## Session Continuity -Last session: 2026-05-04T14:02:52.374Z -Stopped at: Completed 30-01-PLAN.md -Resume file: None +Last session: 2026-05-06T06:55:03.554Z +Stopped at: Phase 35 context gathered +Resume file: .planning/phases/35-ai-discussion-recorder/35-CONTEXT.md diff --git a/.planning/phases/01-config-and-command-foundation/01-LEARNINGS.md b/.planning/phases/01-config-and-command-foundation/01-LEARNINGS.md new file mode 100644 index 00000000..57aeab22 --- /dev/null +++ b/.planning/phases/01-config-and-command-foundation/01-LEARNINGS.md @@ -0,0 +1,318 @@ +--- +phase: 01 +phase_name: "Config and Command Foundation" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 14 + lessons: 5 + patterns: 8 + surprises: 6 +missing_artifacts: + - "UAT.md" +--- + +# Phase 01 Learnings: Config and Command Foundation + +## Decisions + +### D1: Config Precedence Hierarchy (Locked) +The merged config follows a strict layered order: explicit overrides > environment variables > JSON nested `vault_config` > JSON top-level keys > built-in defaults. This was locked as the D-Configuration Hierarchy in Plan 01-01 and never revisited. + +**Rationale:** Multiple config sources (paperforge.json, env vars, CLI arguments) needed an unambiguous resolution order. The hierarchy ensures that a user's `PAPERFORGE_SYSTEM_DIR` env var always beats their paperforge.json setting, and explicit programmatic overrides beat everything. + +**Source:** 01-01-PLAN.md, 01-01-SUMMARY.md + +--- + +### D2: 13-Key Path Inventory (Fixed Contract) +`paperforge_paths()` returns exactly 13 keys: `vault`, `system`, `paperforge`, `exports`, `ocr`, `resources`, `literature`, `control`, `library_records`, `bases`, `worker_script`, `skill_dir`, `ld_deep_script`. `command_dir` was intentionally excluded (not a user-facing diagnostic path). + +**Rationale:** Every downstream consumer (worker, agent, setup, validation, CLI) needed the same path dictionary shape. A fixed contract prevents key drift across 7+ modules. + +**Source:** 01-01-PLAN.md, 01-01-SUMMARY.md + +--- + +### D3: resolve_vault Walks cwd Upward for paperforge.json +The vault root is discovered by searching upward from the current directory for a `paperforge.json` file, enabling `--vault`-free CLI invocation. + +**Rationale:** Reduces friction — users shouldn't need to pass `--vault` every time they run a command from inside their vault. Falls back to explicit `--vault`, then `PAPERFORGE_VAULT` env, then cwd. + +**Source:** 01-01-SUMMARY.md + +--- + +### D4: No os.environ Mutation +The resolver accepts an optional `env` dict parameter and never mutates global `os.environ`. All config loading is a pure function of its inputs. + +**Rationale:** Testability and safety — tests can pass custom env dicts without global side effects. Prevents accidental credential leakage or config mutation across calls. + +**Source:** 01-01-SUMMARY.md + +--- + +### D5: CLI Returns int Exit Codes (Not sys.exit) +`cli.main()` returns an integer exit code rather than calling `sys.exit()` directly. + +**Rationale:** Testability — tests can inspect the return code without catching SystemExit exceptions. The `__main__.py` wrapper is the sole caller of `sys.exit()`. + +**Source:** 01-02-SUMMARY.md + +--- + +### D6: Module-Level Worker Imports in CLI +Worker functions (`run_status`, `run_selection_sync`, etc.) are imported at module level in `cli.py`, not lazily. + +**Rationale:** Tests can monkeypatch worker functions before importing `cli.main`, enabling dispatch verification without invoking real workers. Lazy imports would make patching unreliable. + +**Source:** 01-02-SUMMARY.md + +--- + +### D7: `ocr` Aliases to `ocr run` by Default +The `ocr` subcommand has its default action set to `run`, so `paperforge ocr` and `paperforge ocr run` dispatch identically. + +**Rationale:** Per D-Command Surface requirements — "ocr" alone should trigger the most common operation (running OCR). Reduces user confusion about required sub-subcommands. + +**Source:** 01-02-SUMMARY.md + +--- + +### D8: load_simple_env Added to config.py (Not in Original Scope) +`.env` file loading was added to `paperforge/config.py` during Plan 01-02 because the CLI needed to load environment variables before dispatching to workers. + +**Rationale:** The original Plan 01-01 resolver omitted `.env` loading, but the legacy worker always loaded `.env` files before execution. The CLI needed the same behavior to preserve backward compatibility. + +**Source:** 01-02-SUMMARY.md (Deviations) + +--- + +### D9: Delegate-Wrapper Pattern for Backward Compatibility +Legacy public function names (`load_vault_config`, `pipeline_paths` in worker; `_load_vault_config`, `_paperforge_paths` in ld_deep) were preserved as thin wrappers that delegate to `paperforge.config`. + +**Rationale:** Direct legacy invocation (`python literature_pipeline.py --vault . status`) must continue working. The wrapper pattern keeps the public API stable while centralizing resolver logic. + +**Source:** 01-03-SUMMARY.md + +--- + +### D10: `**shared` Dict Merge for Worker-Only Keys +`pipeline_paths()` uses `**shared` dict merge to combine the shared resolver's 13 keys with worker-only keys (`pipeline`, `candidates`, `search_*`, `harvest_root`, `records`, `review`, `config`, `queue`, `log`, `bridge_config*`, `index`, `ocr_queue`). + +**Rationale:** Worker modules need additional paths beyond the shared contract. The merge approach avoids key collision because shared uses `library_records` while worker uses `records` and `pipeline`. + +**Source:** 01-03-SUMMARY.md + +--- + +### D11: Parallel Package Deployment in Setup Wizard +`setup_wizard.py` copies the `paperforge/` package to two locations: `/worker/paperforge/` and `/literature-qa/paperforge/`. + +**Rationale:** Both the worker script and the ld_deep agent script need to import `paperforge.config`. Copying to both locations ensures either can import the package regardless of which script is invoked. + +**Source:** 01-03-SUMMARY.md + +--- + +### D12: Legacy Fallback in validate_setup.py +`validate_setup.py` tries to import `paperforge.config` first, and falls back to legacy JSON parsing if the shared resolver is unavailable (for pre-Phase-1 installations). + +**Rationale:** Users who haven't yet run `pip install -e .` or had the package deployed by setup_wizard still need validation to work. + +**Source:** 01-03-SUMMARY.md + +--- + +### D13: Stable-Command-First Documentation Strategy +Primary user-facing docs show `paperforge status|selection-sync|index-refresh|ocr run|deep-reading` as the main commands. Legacy `python /PaperForge/worker/scripts/literature_pipeline.py` is retained as a documented fallback with path-resolution instructions. + +**Rationale:** New users should never see unresolved path tokens (``) as their primary way to run commands. But existing users who know the old invocation pattern need continued support. + +**Source:** 01-04-SUMMARY.md + +--- + +### D14: Documentation Test Scope Excludes Architecture Diagrams +`tests/test_command_docs.py` checks user-run code blocks for stable commands but explicitly skips AGENTS.md frontmatter examples, architecture diagrams, and field reference tables. + +**Rationale:** These non-user-run sections legitimately reference internal paths and field names. The Phase 1 scope was to update user-facing commands, not to rewrite the entire architecture documentation. + +**Source:** 01-04-SUMMARY.md + +--- + +## Lessons + +### L1: Missing load_simple_env Was a Planning Gap +During Plan 01-02 implementation, the CLI import failed because `cli.py` needed `load_simple_env` from `config.py`, but Plan 01-01's resolver contract didn't include it. + +**Context:** The legacy worker always called `load_simple_env()` before dispatching commands to load `.env` files. The planning phase didn't trace this dependency fully. Auto-fixed during implementation by adding the function to config.py. + +**Source:** 01-02-SUMMARY.md (Deviations section) + +--- + +### L2: Standalone Scripts Require Special Import Handling +`literature_pipeline.py` and `ld_deep.py` are standalone scripts (no `__init__.py`), which meant tests couldn't use standard `import` statements. Required `importlib.util.spec_from_file_location` to load them as modules. + +**Context:** Tests needed to import these scripts to verify their wrapper functions delegate to the shared resolver, but Python's import system treats them as scripts, not modules. + +**Source:** 01-03-SUMMARY.md (Issues Encountered) + +--- + +### L3: Subprocess Tests Need Explicit PYTHONPATH +The subprocess smoke test (`python literature_pipeline.py --vault . status`) initially failed with `ModuleNotFoundError: No module named 'paperforge'` because `paperforge` wasn't on the subprocess's Python path. + +**Context:** The test needed to simulate the installed-package scenario. Fixed by passing `PYTHONPATH` env var to the subprocess, which mirrors how an editable install would make the package available. + +**Source:** 01-03-SUMMARY.md (Issues Encountered) + +--- + +### L4: TDD-for-Docs Creates Temporary State Where Docs Reference Non-Existent Commands +Plan 01-04 wrote documentation tests AND updated docs in the same plan, but the SUMMARY noted that "the actual CLI module does not exist in this repo yet" — documentation was updated before the CLI was fully implemented across all phases. + +**Context:** The TDD-for-docs pattern ensures tests catch regressions, but the doc content itself references commands that were only partially wired at the time of writing. + +**Source:** 01-04-SUMMARY.md (Next Phase Readiness) + +--- + +### L5: Well-Scoped Plans Execute with Near-Zero Deviations +All 4 sub-plans executed essentially as written. Plans 01-01, 01-03, and 01-04 had zero deviations from the plan. Plan 01-02 had one auto-fixed blocking deviation (L1 above). This suggests the planning phase had high precision. + +**Context:** This was the first phase of the PaperForge release hardening project. The high planning accuracy established confidence in the GSD methodology for subsequent phases. + +**Source:** All four SUMMARY.md files + +--- + +## Patterns + +### P1: Delegate-Wrapper Pattern +Preserve existing public API function names but replace their implementation bodies with thin wrappers that delegate to the shared module. The function signatures stay identical; only the internals change. + +**When to use:** When migrating existing code to a shared library while preserving backward compatibility for direct invocation and existing callers. + +**Source:** 01-03-SUMMARY.md + +--- + +### P2: Parallel Package Deployment +Copy the shared utility package to each script's adjacent directory so that standalone scripts in different directory trees can all import the same library without requiring pip installation or PYTHONPATH manipulation. + +**When to use:** When deploying a Python package to a vault or runtime environment where `pip install` is not always available or reliable. + +**Source:** 01-03-SUMMARY.md + +--- + +### P3: stdlib-Only Layered Merge Resolver +Build configuration resolution using only stdlib (`json`, `os`, `pathlib`) with a deterministic layered merge: hardcoded defaults → JSON file → environment variables → explicit overrides. Each layer overwrites keys from previous layers. + +**When to use:** For local-first CLI tools that need configurable paths with clear precedence, without pulling in config management libraries. + +**Source:** 01-01-SUMMARY.md + +--- + +### P4: Upward-Directory Vault Discovery +Discover the project root by searching upward from cwd for a sentinel file (`paperforge.json`). Fall back through explicit CLI argument, then environment variable, then cwd itself. + +**When to use:** For CLI tools that operate within a project directory but may be invoked from any subdirectory. + +**Source:** 01-01-SUMMARY.md + +--- + +### P5: Fixed Command-to-Function Dispatch Map +Use a static dictionary mapping command name strings to handler functions rather than dynamically evaluating command names or constructing shell commands. + +**When to use:** Any CLI that dispatches user-provided subcommand names to handler functions — mitigates code injection and improves auditability. + +**Source:** 01-02-PLAN.md, 01-02-SUMMARY.md + +--- + +### P6: paths --json as Machine-Parseable Contract +The `paths --json` subcommand outputs only the key contract keys (not all internal paths), producing valid JSON that scripts and tools can consume programmatically. + +**When to use:** When CLI output needs to be consumed by automation scripts, CI pipelines, or agent helpers that need resolved paths. + +**Source:** 01-02-SUMMARY.md + +--- + +### P7: Stable-Command-First Documentation +In user-facing docs, present the new stable launcher command as the primary example, with the legacy invocation shown as a clearly labeled fallback with path-resolution instructions using `paperforge paths --json`. + +**When to use:** When migrating documentation from a legacy command surface to a new one — gives new users the clean path while supporting existing users. + +**Source:** 01-04-SUMMARY.md + +--- + +### P8: TDD for Documentation +Write tests that assert specific content must exist (or must NOT exist) in documentation files before editing the docs. This creates a regression safety net that catches documentation drift in future changes. + +**When to use:** When documentation is critical enough that regressions (unresolved tokens, outdated commands) would break the user experience. + +**Source:** 01-04-SUMMARY.md + +--- + +## Surprises + +### S1: All Config Tests Passed on First Implementation Attempt +After writing 22 failing tests (TDD RED), the full `config.py` implementation passed all 22 on the first run. No iterative debugging or refactoring was needed. + +**Impact:** Plan 01-01 completed in just 8 minutes. The planning phase (RESEARCH.md, CONTEXT.md, D-Configuration Hierarchy) was thorough enough that the implementation was essentially transcription. + +**Source:** 01-01-SUMMARY.md + +--- + +### S2: load_simple_env Blocking Gap Escaped Planning Phase +The `.env` file loading function was omitted from the Plan 01-01 resolver contract but was required by Plan 01-02's CLI implementation. This dependency was not traced during the multi-source coverage audit. + +**Impact:** Plan 01-02 had its only deviation — a blocking auto-fix that added the function to config.py. The planning audit tracked 9 requirement-to-plan mappings but missed the `.env` → CLI dependency chain. + +**Source:** 01-02-SUMMARY.md (Deviations) + +--- + +### S3: Script-vs-Module Import Distinction Required Test Adaptation +The worker and ld_deep scripts are standalone executables without `__init__.py`, which meant standard `import` statements couldn't load them in tests. Required `importlib.util.spec_from_file_location` workaround. + +**Impact:** Moderate test code complexity — the import pattern is non-standard and less readable than normal imports. This is a structural issue that could be revisited if these scripts are converted to proper modules. + +**Source:** 01-03-SUMMARY.md (Issues Encountered) + +--- + +### S4: Subprocess Environment Isolation Was an Unanticipated Test Requirement +The legacy worker invocation smoke test required explicit `PYTHONPATH` injection because subprocesses don't inherit the test process's Python path configuration. + +**Impact:** This wasn't described in the test design but was caught immediately during implementation. The fix was straightforward (pass `PYTHONPATH` env to subprocess), but the need for it wasn't anticipated. + +**Source:** 01-03-SUMMARY.md (Issues Encountered) + +--- + +### S5: Full Phase Completed in ~38 Minutes — Faster Than Expected +The combined duration of all 4 sub-plans was approximately 38 minutes (8 + 11 + 15 + 4.5). For a phase touching 7 modules and producing 58 tests across 6 test files, this was faster than typical phase estimates. + +**Impact:** Demonstrated that the GSD planning methodology (RESEARCH → CONTEXT → PLAN → EXECUTE) with clear dependency graphs between sub-plans produces high-velocity execution. The sequential dependency chain (01→02→03→04) didn't create bottlenecks because each plan was independently scoped. + +**Source:** All four SUMMARY.md duration metrics + +--- + +### S6: 01-VERIFICATION.md Confirmed 8/8 Truths with Zero Gaps +The post-hoc verification scan found that all 8 observable truths were satisfied, all 14 required artifacts existed, all 9 key links were wired, and all 8 requirements were satisfied. Zero anti-patterns or gaps were found. + +**Impact:** The phase achieved 100% requirement coverage with no deferred items, no TODO placeholders, and no partial implementations. This level of completeness was not guaranteed at the planning stage. + +**Source:** 01-VERIFICATION.md diff --git a/.planning/phases/02-paddleocr-and-pdf-path-hardening/02-LEARNINGS.md b/.planning/phases/02-paddleocr-and-pdf-path-hardening/02-LEARNINGS.md new file mode 100644 index 00000000..2c87a6fd --- /dev/null +++ b/.planning/phases/02-paddleocr-and-pdf-path-hardening/02-LEARNINGS.md @@ -0,0 +1,247 @@ +--- +phase: 02 +phase_name: "PaddleOCR and PDF Path Hardening" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 12 + lessons: 4 + patterns: 5 + surprises: 4 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +# Phase 02 Learnings: PaddleOCR and PDF Path Hardening + +## Decisions + +### D1: `nopdf` as a Distinct Terminal State +Missing or unreadable PDFs result in `ocr_status: nopdf` — distinct from `blocked` (fixable config/path issues) and `error` (runtime/API issues). This gives users immediate visibility into which records lack PDFs before queueing OCR. + +**Rationale:** `nopdf` means "no PDF available to OCR" — user should check Zotero attachment. It prevents wasted OCR runs on attachments that don't exist. + +**Source:** 02-01-PLAN.md, 02-01-SUMMARY.md, 02-CONTEXT.md + +--- + +### D2: Junction Resolution Strategy +Windows junctions are resolved via `os.path.realpath` first, with a fallback to `GetFinalPathNameByHandleW` via ctypes when `realpath` does not follow the junction. + +**Rationale:** `os.path.realpath` handles most symlinks but may not resolve all Windows directory junctions. The ctypes fallback provides comprehensive coverage. + +**Source:** 02-01-PLAN.md, 02-01-SUMMARY.md + +--- + +### D3: `resolve_pdf_path` Resolution Order +Path resolution tries strategies in strict sequence: absolute path → vault-relative path → junction resolution → Zotero storage-relative path. Returns empty string if all fail. + +**Rationale:** Supports all known Zotero attachment path formats while providing predictable fallback behavior. Early exit on first successful resolution. + +**Source:** 02-01-PLAN.md, 02-01-SUMMARY.md + +--- + +### D4: Failure Taxonomy — `blocked` vs `error` +All OCR failures are classified into two categories: `blocked` (fixable issues like config, path, or token problems) and `error` (runtime issues like API timeout, schema mismatch, provider error). + +**Rationale:** Enables actionable user messaging — blocked items tell the user exactly what to fix; error items signal a transient or provider-side issue. + +**Source:** 02-02-PLAN.md, 02-02-SUMMARY.md, 02-CONTEXT.md + +--- + +### D5: `suggestion` Field in `meta.json` +Every failure path writes a `suggestion` field to `meta.json` containing an actionable fix instruction for the user. + +**Rationale:** Eliminates the opaque "PaddleOCR request failed" debugging experience. Users see exactly what to do next. + +**Source:** 02-02-PLAN.md, 02-02-SUMMARY.md + +--- + +### D6: Tiered L1-L4 Diagnostics with Early-Stop +OCR Doctor runs diagnostics in tiered order: L1 (token presence) → L2 (URL reachability) → L3 (API response structure) → L4 (live PDF test, optional). Stops at the first failure with actionable output. + +**Rationale:** Each level depends on the previous. No point testing API schema if the URL is unreachable. Early-stop minimizes unnecessary API calls. + +**Source:** 02-03-PLAN.md, 02-03-SUMMARY.md + +--- + +### D7: L4 Live PDF Test Is Optional (`--live`) +The full round-trip PDF upload test (L4) requires the `--live` flag explicitly. Without it, doctor stops at L3. + +**Rationale:** L4 consumes API resources and may hit rate limits. Making it opt-in prevents accidental resource waste while still providing the option for thorough diagnosis. + +**Source:** 02-03-PLAN.md, 02-03-SUMMARY.md + +--- + +### D8: CLI Sub-Subcommands for OCR (`ocr run`, `ocr doctor`) +The CLI changed from a flat `ocr` command to sub-subcommands: `paperforge ocr run` and `paperforge ocr doctor`. `ocr` without a subcommand defaults to `run`. + +**Rationale:** Multiple OCR-related operations needed distinct entry points. Sub-subcommands provide clear separation while preserving backward compatibility through the default. + +**Source:** 02-03-PLAN.md, 02-03-SUMMARY.md + +--- + +### D9: Inline Import of `classify_error` in `run_ocr()` +`classify_error` is imported inline (inside the except block) rather than at the top of the file. + +**Rationale:** Avoids circular import issues in the test environment where `run_ocr()` may be imported before `ocr_diagnostics.py` is fully available. + +**Source:** 02-02-PLAN.md, 02-02-SUMMARY.md + +--- + +### D10: `raw_response` Truncated to 1000 Characters +On polling schema mismatch, the raw API response is stored in `meta.json` truncated to 1000 characters for safety. + +**Rationale:** Raw responses may contain large payloads. Truncation prevents meta.json bloat while preserving enough content for debugging schema changes. + +**Source:** 02-02-PLAN.md, 02-02-SUMMARY.md + +--- + +### D11: `record_ocr_status` Variable in Selection-Sync +`run_selection_sync()` introduces a local `record_ocr_status` variable (distinct from the existing `ocr_status` variable used for meta.json state) to hold the frontmatter-only status value. + +**Rationale:** Avoids confusing the library-record frontmatter OCR status with the meta.json OCR state. The frontmatter value is derived from preflight checks and may differ from the persistent meta.json state. + +**Source:** 02-04-PLAN.md, 02-04-SUMMARY.md + +--- + +### D12: Explicit Boolean Conversion in `yaml_quote` +The `yaml_quote()` function was fixed to handle Python `True`/`False` booleans by converting them to YAML `true`/`false` instead of treating them as falsy empty strings. + +**Rationale:** Python `bool(True)` was falling through to the falsy branch, causing frontmatter corruption for `has_pdf`, `do_ocr`, and `analyze` fields. + +**Source:** 02-04-SUMMARY.md + +--- + +## Lessons + +### L1: Windows Junction Mock Test Is Unpatchable +The test for Windows directory junction resolution is skipped because `from ctypes import wintypes` inside `resolve_junction()` bypasses module-level `ctypes` patches due to Python import semantics when `wintypes` is imported inside a function body. + +**Context:** The symlink test and `os.path.realpath` path provide adequate coverage, but the direct ctypes junction code path cannot be unit-tested on non-Windows platforms. + +**Source:** 02-01-SUMMARY.md + +--- + +### L2: Exact String Matching in Test Assertions +One test assertion initially failed because it matched "format changed" vs "schema" — the expected suggestion string differed slightly from the implementation. + +**Context:** The `classify_error` function for `JSONDecodeError` contained "PaddleOCR API response format changed." in the plan but "PaddleOCR API response schema changed." in the implementation (from earlier code). Ensures test assertions must match implementation exactly. + +**Source:** 02-02-SUMMARY.md + +--- + +### L3: Registry Token Blocks Env-Based Test Patters +The PaddleOCR registry token is always present in the test environment, making `patch.dict(os.environ, {}, clear=True)` ineffective for simulating a missing token scenario. + +**Context:** Tests that need to verify behavior when `PADDLEOCR_API_TOKEN` is missing must use alternative approaches like HTTP 401 error injection rather than env var manipulation. + +**Source:** 05-01-SUMMARY.md + +--- + +### L4: Shared Dict Mutation with `return_value` in Mocks +Patching `ensure_ocr_meta` with `return_value={}` caused shared dict mutation because the same dict object was reused across multiple items in the loop. + +**Context:** Must use `side_effect` with a factory function returning a fresh dict on each call instead of `return_value` when the mocked function is called multiple times and the caller mutates the result. + +**Source:** 05-01-SUMMARY.md + +--- + +## Patterns + +### P1: Path Resolution Fallback Chain +Multiple resolution strategies tried in a fixed sequence, returning the first successful result. Each strategy handles a distinct path format. Empty string signals total failure. + +**When to use:** When a value can be expressed in multiple formats and you need to normalize to a canonical form, with clear fallback semantics. + +**Source:** 02-01-PLAN.md, 02-01-SUMMARY.md + +--- + +### P2: Inline Import for Testability +Import a module inline (inside a function body, not at file top-level) to break circular import chains that manifest only during test invocation. + +**When to use:** When two modules cyclically depend on each other at import time, or when a standalone script imports a module that itself imports back into the script's package. + +**Source:** 02-02-PLAN.md, 02-02-SUMMARY.md + +--- + +### P3: Classify-and-Suggest Error Handling +Map exceptions to (state, suggestion) pairs via a centralized classification function. The state drives downstream behavior; the suggestion drives user-facing messaging. + +**When to use:** When different error types require different recovery strategies and different user-facing messages, but all errors flow through a single handler. + +**Source:** 02-02-PLAN.md, 02-02-SUMMARY.md + +--- + +### P4: Tiered Diagnostics with Early-Stop +Run checks in dependency order, stopping at the first failure. Each level validates a prerequisite for the next. Output is always actionable and specific to the failing level. + +**When to use:** Diagnostic tools where higher-level checks are meaningless without lower-level prerequisites. Avoids users seeing "API schema mismatch" when the real issue is a missing API key. + +**Source:** 02-03-PLAN.md, 02-03-SUMMARY.md + +--- + +### P5: Worker Preflight Pattern +Before performing an operation, validate all prerequisites (file existence, path resolution, state validity). On preflight failure, record a specific terminal state and skip the operation. + +**When to use:** Worker functions that operate on external resources (files, APIs, network services) where failure after partial work would leave inconsistent state. + +**Source:** 02-01-PLAN.md, 02-04-PLAN.md + +--- + +## Surprises + +### S1: ctypes Import Inside Function Body Bypasses Module-Level Patches +`from ctypes import wintypes` inside `resolve_junction()` creates a new reference to the ctypes module that cannot be intercepted by patching `paperforge.pdf_resolver.ctypes` at module level. + +**Impact:** The Windows junction mock test had to be disabled (skipped) on non-Windows platforms. The symlink test and `os.path.realpath` path provide equivalent coverage but the ctypes path remains untested on non-Windows. + +**Source:** 02-01-SUMMARY.md + +--- + +### S2: `yaml_quote()` Didn't Handle Python Boolean `True`/`False` +Python's `bool(True)` was treated as a falsy value by the YAML quoting function, producing empty strings for `has_pdf`, `do_ocr`, and `analyze` fields in frontmatter. + +**Impact:** Introduced a subtle frontmatter corruption bug that silently dropped boolean fields. Fixed by adding explicit `isinstance(v, bool)` check before the truthiness test. This also fixed boolean handling in other frontmatter update paths. + +**Source:** 02-04-SUMMARY.md + +--- + +### S3: Obsidian Base UI Renders Absolute Paths for `pdf_path` +Obsidian's Base view displays `pdf_path` as an absolute path in the UI even though the data is stored as a relative wikilink in the actual file. + +**Impact:** Cosmetic issue only — does not affect functionality. Users may be confused by the display but the underlying data is correct. Not worth fixing. + +**Source:** AGENTS.md (FAQ section) + +--- + +### S4: Chinese Domain Names Pass Through `slugify` Unchanged +`slugify_filename()` does not transliterate CJK characters, so Chinese domain names like `骨科` produce `骨科.base` directly rather than being transliterated to `guke.base`. + +**Impact:** Correct behavior for this project — users want Chinese filenames to remain Chinese. But surprising if the developer assumed slugify would produce ASCII-only filenames. + +**Source:** 03-01-SUMMARY.md diff --git a/.planning/phases/03-obsidian-bases-config-aware/03-LEARNINGS.md b/.planning/phases/03-obsidian-bases-config-aware/03-LEARNINGS.md new file mode 100644 index 00000000..94441221 --- /dev/null +++ b/.planning/phases/03-obsidian-bases-config-aware/03-LEARNINGS.md @@ -0,0 +1,175 @@ +--- +phase: 03 +phase_name: "Obsidian Bases Config-Aware" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 9 + lessons: 3 + patterns: 3 + surprises: 2 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +# Phase 03 Learnings: Obsidian Bases Config-Aware + +## Decisions + +### D1: Exactly 8 Views Per Domain Base File +Each domain Base file contains exactly 8 views: 控制面板, 推荐分析, 待 OCR, OCR 完成, 待深度阅读, 深度阅读完成, 正式卡片, 全记录. + +**Rationale:** These 8 views map directly to the production workflow pipeline stages. Any fewer would miss workflow states; any more would add unnecessary complexity for this version. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### D2: Config-Aware Path Placeholders (`${LIBRARY_RECORDS}`, etc.) +Base files use `${LIBRARY_RECORDS}`, `${LITERATURE}`, and `${CONTROL_DIR}` placeholders in `folder_filter` expressions instead of hardcoded paths. + +**Rationale:** The user can configure custom directory names in `paperforge.json`. Placeholders ensure generated Base files always use the correct paths regardless of configuration. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### D3: `# PAPERFORGE_VIEW:` Marker Prefix for View Identification +Each PaperForge-generated view is preceded by a comment marker `# PAPERFORGE_VIEW: ` in the .base file for unambiguous identification. + +**Rationale:** The incremental merge mechanism needs to know which views are PaperForge-owned vs user-defined. The marker provides a stable, parseable identifier that survives content changes. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### D4: Incremental Merge — Replace PF Views, Preserve User Views +On refresh (default): PaperForge standard views are always replaced with fresh content. User-defined views (those without the PAPERFORGE_VIEW prefix) are never touched. + +**Rationale:** PaperForge must keep its 8 standard views aligned with the current workflow definition. User customizations to non-standard views are valuable and must survive refreshes. User customizations to standard view filters are intentionally reverted (trade-off). + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### D5: `--force` Flag for Full Regeneration +The `base-refresh --force` flag bypasses merge entirely and regenerates all views from scratch, wiping any user customizations. + +**Rationale:** Provides an escape hatch when the Base file becomes corrupted or the user wants to reset to factory defaults. Without this, there is no way to clean up a broken Base file. + +**Source:** 03-01-PLAN.md, 03-02-PLAN.md + +--- + +### D6: Literature Hub.base and PaperForge.base in Addition to Per-Domain Bases +Three types of Base files are generated: one per domain (e.g., `骨科.base`), a cross-domain `Literature Hub.base` (filters on `${LIBRARY_RECORDS}` root), and a legacy `PaperForge.base` (all records). + +**Rationale:** Different users need different views of the data — per-domain for focused work, cross-domain for overview, and an all-records view for complete inventory. + +**Source:** 03-01-SUMMARY.md + +--- + +### D7: `ensure_base_views` Accepts `force: bool = False` Parameter +The function signature was updated from `ensure_base_views(vault, paths, config)` to `ensure_base_views(vault, paths, config, force=False)`. + +**Rationale:** The inner `refresh_base` function branches on `force`: when False, calls `merge_base_views` for incremental merge; when True, calls `_build_base_yaml` directly for full regeneration. + +**Source:** 03-02-PLAN.md + +--- + +### D8: `folder_filter` Uses Placeholder Format, Substituted After Merge +The `folder_filter` parameter (e.g., `${LIBRARY_RECORDS}/骨科`) is passed in placeholder format and only substituted after merge is complete. + +**Rationale:** Merging operates on raw YAML content, not resolved paths. Substituting after merge ensures the merge logic works with stable placeholder strings, not machine-specific paths. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### D9: STANDARD_VIEW_NAMES as a `frozenset` +The 8 standard view names are defined as a module-level `frozenset` for fast membership checks in the merge logic. + +**Rationale:** `merge_base_views()` needs to identify PaperForge views by name. A frozenset provides O(1) lookup and immutability guarantees. + +**Source:** 03-01-PLAN.md + +--- + +## Lessons + +### L1: User Filter Modifications on Standard Views Are Reverted +If a user modifies the filter expression of a standard PaperForge view (e.g., changes the "OCR 完成" filter), the next incremental refresh reverts it to the template definition. + +**Context:** This is an intentional trade-off — standard views must match the current workflow definition. User customizations to standard views are not preserved. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### L2: `slugify_filename()` Does Not Transliterate CJK Characters +Chinese characters like `骨科` pass through unchanged, producing `骨科.base` rather than being transliterated to `guke.base`. + +**Context:** The slugify function handles whitespace and special characters but does not convert CJK to Latin. This is acceptable for this project but was surprising during initial verification. + +**Source:** 03-01-SUMMARY.md + +--- + +### L3: `folder_filter` Must Use Placeholder Format Substituted After Merge +Placeholders must survive the merge operation. If paths were substituted before merge, the merge logic would see machine-specific absolute paths and fail to match views correctly. + +**Context:** The merge strategy depends on stable, predictable content to identify and replace PaperForge views. Resolved paths vary between machines and would break merge matching. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +## Patterns + +### P1: Incremental Merge with Marker Comments +Use a stable comment prefix to mark owned content. On merge, replace all marked content with fresh versions while preserving everything else. The prefix serves as both documentation and merge boundary. + +**When to use:** When multiple writers (a tool and a user) modify the same structured file and the tool needs to update its own sections without clobbering user additions. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### P2: Template Placeholder Substitution at Generation Time +Use `${SCREAMING_SNAKE_CASE}` tokens in template content. Resolve to real values at write time based on configuration. Unrecognized placeholders are left unchanged. + +**When to use:** When generated content must reference configurable paths, names, or values that are only known at generation time, not at template authoring time. + +**Source:** 03-01-PLAN.md, 03-01-SUMMARY.md + +--- + +### P3: Shared Properties YAML Template +Define the properties section (which is identical across all Base files) as a single YAML template string, reused by all Base file generators. + +**When to use:** When multiple generated files share a common structured section. Avoids duplication and ensures consistent field definitions across all outputs. + +**Source:** 03-01-PLAN.md + +--- + +## Surprises + +### S1: Chinese Filenames Pass Through `slugify` Unchanged +The `slugify_filename()` function does not transliterate CJK characters, producing `骨科.base` directly instead of `guke.base`. + +**Impact:** Domain Base files use Chinese filenames. This is correct behavior for this project but was surprising because typical slugify implementations convert all non-ASCII characters. No functional issue. + +**Source:** 03-01-SUMMARY.md + +--- + +### S2: 120 Tests Pass Without Regression +The Base generation refactor (touching ~300 lines of core logic) did not break any existing tests. All 120 tests pass, including 2 skipped junction tests. + +**Impact:** Demonstrates that the test suite provides good regression coverage. The incremental merge system was complex enough that some test failures were expected but none occurred. + +**Source:** 03-01-SUMMARY.md, 03-02-SUMMARY.md diff --git a/.planning/phases/04-onboarding-validation/04-LEARNINGS.md b/.planning/phases/04-onboarding-validation/04-LEARNINGS.md new file mode 100644 index 00000000..0b4b507f --- /dev/null +++ b/.planning/phases/04-onboarding-validation/04-LEARNINGS.md @@ -0,0 +1,145 @@ +--- +phase: 04 +phase_name: "Onboarding Validation" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 8 + lessons: 3 + patterns: 2 + surprises: 0 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +# Phase 04 Learnings: Onboarding Validation + +## Decisions + +### D1: Three-State `deep-reading` Output +`paperforge deep-reading` outputs three distinct sections: 就绪 (ready — OCR done), 等待 OCR (waiting — `do_ocr=true` and `ocr_status=pending/processing`), 阻塞 (blocked — `analyze=true` but OCR not done and not waiting). + +**Rationale:** Users need to see at a glance which papers are ready for deep reading, which are waiting for OCR, and which are blocked by missing OCR configuration. The three-state breakdown prevents confusion between these fundamentally different states. + +**Source:** 04-01-PLAN.md, 04-01-SUMMARY.md + +--- + +### D2: `--verbose` Flag Shows Fix Instructions for Blocked Papers +When `--verbose` is passed, each blocked paper shows a specific fix instruction based on its `ocr_status`: pending → "run `paperforge ocr run`", processing → "wait for completion", failed → "check meta.json then retry". + +**Rationale:** Blocked papers are the user's action items. Verbose mode eliminates guesswork by telling the user exactly what to do for each blocked paper. + +**Source:** 04-01-PLAN.md, 04-01-SUMMARY.md + +--- + +### D3: Doctor Checks 7 Categories +`paperforge doctor` validates 7 categories: Python environment, Vault structure, Zotero link, BBT export, OCR configuration, Worker scripts, Agent scripts. + +**Rationale:** Covers the full installation and configuration surface. Each category is independently checkable and has a clear pass/fail/warn outcome. + +**Source:** 04-02-PLAN.md, 04-02-SUMMARY.md + +--- + +### D4: Doctor Output Format `[PASS]/[FAIL]/[WARN]` with Fix Suggestions +Each check category prints `[PASS]` / `[FAIL]` / `[WARN]` with a human-readable message. On failure, a "修复步骤" (fix steps) section follows with actionable instructions. + +**Rationale:** Users need to quickly scan which checks passed and which need attention. The consistent prefix enables visual scanning while fix steps provide guidance. + +**Source:** 04-02-PLAN.md, 04-02-SUMMARY.md + +--- + +### D5: `run_deep_reading` Modified to Accept `verbose: bool = False` +The function signature was changed from `run_deep_reading(vault: Path)` to `run_deep_reading(vault: Path, verbose: bool = False)`. + +**Rationale:** Adding an optional parameter with a default value maintains backward compatibility with existing call sites while enabling the new verbose behavior. + +**Source:** 04-01-PLAN.md, 04-01-SUMMARY.md + +--- + +### D6: AGENTS.md Uses `paperforge ` as Primary Invocation +All command examples in AGENTS.md were updated to show `paperforge ` as the primary invocation, with legacy `python ... literature_pipeline.py` paths retained as commented backup. + +**Rationale:** New users should never see unresolved path tokens as their primary command reference. Existing users who know the old pattern still have a documented fallback. + +**Source:** 04-03-PLAN.md, 04-03-SUMMARY.md + +--- + +### D7: `docs/README.md` as User-Facing BBT Configuration Guide +Created as a separate user-facing document (distinct from AGENTS.md which is agent-facing) containing step-by-step Better BibTeX configuration instructions in Chinese. + +**Rationale:** Different audiences need different documentation. Users configuring BBT for the first time need a linear step-by-step guide; agents need a command reference. + +**Source:** 04-04-PLAN.md, 04-04-SUMMARY.md + +--- + +### D8: `_is_junction()` Helper Function for Doctor Checks +Added a dedicated helper function `_is_junction(path: Path) -> bool` using Windows reparse point detection for the Zotero link validity check. + +**Rationale:** The doctor needs to distinguish between a proper junction/symlink (recommended) and a regular directory copy (warn). The helper encapsulates platform-specific detection logic. + +**Source:** 04-02-SUMMARY.md + +--- + +## Lessons + +### L1: Three-State Deep-Reading Logic Has Overlapping Categories +A paper with `do_ocr=true`, `analyze=true`, and `ocr_status=pending` is simultaneously "waiting for OCR" AND "blocked for deep reading." The output must handle this overlap without double-counting. + +**Context:** The implementation logic uses a priority: ready (done) > waiting OCR (do_ocr + pending/processing) > blocked (analyze + not done and not waiting). An item appears in exactly one category based on the first matching condition. + +**Source:** 04-01-PLAN.md + +--- + +### L2: Legacy Commands Must Be Retained as Commented Backup +Users who installed PaperForge before the CLI unified entry point may not have `paperforge` registered as a command. These users need the legacy `python ...` path as a fallback. + +**Context:** The command consistency check in Plan 05-01 confirmed that AGENTS.md properly retains legacy paths as commented fallbacks. Removing them entirely would break existing users. + +**Source:** 04-03-PLAN.md, 05-01-SUMMARY.md + +--- + +### L3: Handler Function Signature Evolution Requires Test Stub Updates +Adding the `verbose` parameter to `run_deep_reading` required updating the test stub `stub_run_deep_reading` to accept `verbose: bool = False` for backward compatibility. + +**Context:** The CLI dispatch test stubs must match the worker function signatures. Adding a parameter to one requires updating all call sites in tests as well. + +**Source:** 04-01-SUMMARY.md + +--- + +## Patterns + +### P1: Doctor Categorized Check Pattern +Each check category is implemented as an independent function returning a tuple of (status, message, fix_suggestion). The main `run_doctor` function iterates over all checks and collects results into a grouped report. + +**When to use:** Validation/diagnostic tools that need to check multiple independent aspects of a system with clear pass/fail outcomes and actionable fixes. + +**Source:** 04-02-PLAN.md, 04-02-SUMMARY.md + +--- + +### P2: Handler Function Evolution with Optional Parameters +Adding new behavior to an existing function by adding optional parameters with default values rather than creating a new function or changing required parameters. + +**When to use:** When extending an existing function's behavior without breaking existing callers. The default value should match the old behavior for backward compatibility. + +**Source:** 04-01-PLAN.md, 04-01-SUMMARY.md + +--- + +## Surprises + +No significant surprises were encountered during Phase 04. All four sub-plans executed as written with no deviations, no blocking issues, and no unexpected findings. The work was more mechanical (updating docs, adding straightforward validation commands) than the preceding phases, contributing to the lack of surprises. + +**Source:** All four SUMMARY.md files (no deviations or issues sections) diff --git a/.planning/phases/05-workflow-hardening-and-optional-plugin-shell/05-LEARNINGS.md b/.planning/phases/05-workflow-hardening-and-optional-plugin-shell/05-LEARNINGS.md new file mode 100644 index 00000000..b730e9d9 --- /dev/null +++ b/.planning/phases/05-workflow-hardening-and-optional-plugin-shell/05-LEARNINGS.md @@ -0,0 +1,175 @@ +--- +phase: 05 +phase_name: "Workflow Hardening and Optional Plugin Shell" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 7 + lessons: 3 + patterns: 4 + surprises: 3 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +# Phase 05 Learnings: Workflow Hardening and Optional Plugin Shell + +## Decisions + +### D1: OCR State Machine Tests Cover 7 States +The state machine tests cover: pending, queued, running, done, error, blocked, and nopdf — all 7 OCR job states. + +**Rationale:** Comprehensive coverage of all possible OCR job lifecycle states ensures that state transitions are handled correctly and no edge case crashes the worker. + +**Source:** 05-01-PLAN.md, 05-01-SUMMARY.md + +--- + +### D2: Blocked State Triggered via HTTPError 401 — Not Env Manipulation +Instead of patching environment variables to simulate a missing token (which doesn't work due to always-present registry tokens), the blocked state is triggered by making `requests.post` raise `HTTPError(401)` with `response.status_code=401`. + +**Rationale:** The registry token is always present in the test environment, making `patch.dict(os.environ, {}, clear=True)` ineffective. HTTP 401 directly triggers the `classify_error` path that produces the `'blocked'` state. + +**Source:** 05-01-SUMMARY.md + +--- + +### D3: `ensure_ocr_meta` Patched with `side_effect` Factory (Not `return_value`) +The `ensure_ocr_meta` function is patched with `side_effect=make_meta` where `make_meta` is a factory function returning a fresh dict on each call, rather than `return_value={}` which would reuse the same dict. + +**Rationale:** When called multiple times in a loop, the caller mutates the returned dict. A shared dict causes all subsequent iterations to see accumulated mutations. + +**Source:** 05-01-SUMMARY.md + +--- + +### D4: 13 End-to-End Smoke Tests Cover Full Pipeline +The smoke test suite (`test_smoke.py`) contains 13 tests across 6 categories: setup validation, selection sync, index refresh, OCR doctor (L1-L3 mocked), deep-reading queue, and CLI main entry. + +**Rationale:** Ensures the entire pipeline — from `paperforge doctor` setup validation through `deep-reading` queue — can execute without crashing, using mocked external dependencies. + +**Source:** 05-02-PLAN.md, 05-02-SUMMARY.md + +--- + +### D5: Smoke Tests Use `tmp_path` Fixture Vault — Never Real Vault +Every smoke test creates a complete vault structure under `tmp_path` using the `fixture_vault` factory fixture. No test accesses the real user vault. + +**Rationale:** Test isolation and safety — a bug in a test should never affect the user's real data. The fixture vault includes paperforge.json, directory structure, mock library records, and BBT export JSON. + +**Source:** 05-02-PLAN.md, 05-02-SUMMARY.md + +--- + +### D6: All Network Calls Mocked — No Live API in Tests +OCR diagnostics tests mock `requests.get` and `requests.post` via `unittest.mock.patch` to verify the full diagnostic flow without hitting the real PaddleOCR API. + +**Rationale:** Tests must be hermetic and fast. Live API calls would introduce network dependency, rate limits, and timing issues into the test suite. + +**Source:** 05-02-PLAN.md, 05-02-SUMMARY.md + +--- + +### D7: Fixture Library Records Use `TESTKEY001/002/003` +Mock library records use predictable zotero_keys (`TESTKEY001`, `TESTKEY002`, `TESTKEY003`) that are easy to debug in test output. + +**Rationale:** Predictable test data makes test failures easier to diagnose. The sequential key pattern also verifies that the system handles multiple records correctly in batch operations. + +**Source:** 05-02-PLAN.md, 05-02-SUMMARY.md + +--- + +## Lessons + +### L1: Registry Token Makes Env-Based Test Patterns Unreliable +The PaddleOCR registry token is always present in the test environment. Patching `os.environ` to remove it has no effect on the actual token resolution path (which reads from Windows registry). + +**Context:** This required 7 attempts to fix `test_missing_token_blocks_job` before finding the working approach: inject HTTPError 401 as a side_effect on the mocked `requests.post` to trigger the `classify_error` → `'blocked'` mapping. + +**Source:** 05-01-SUMMARY.md + +--- + +### L2: `ensure_ocr_meta` Dict Mutation Requires Factory Pattern +When `ensure_ocr_meta` is patched with `return_value={}`, all callers in the loop receive the same dict object. Mutations by one iteration affect all subsequent iterations. + +**Context:** The run_ocr function iterates over queue items, each time calling `ensure_ocr_meta()` and then mutating the returned dict. Using `side_effect` with a factory creates independent dicts per call. + +**Source:** 05-01-SUMMARY.md + +--- + +### L3: `cleanup_blocked_ocr_dirs` Has Selective Cleanup Behavior +The cleanup function does NOT blindly remove all blocked directories — it preserves directories that contain a `fulltext.md` payload (indicating partial previous OCR results) and only removes empty blocked directories. + +**Context:** This selective behavior prevents data loss where a blocked job may have produced partial results. The test suite includes tests for both cases (preserve vs remove) to lock this behavior. + +**Source:** 05-01-SUMMARY.md + +--- + +## Patterns + +### P1: Fixture Factory Pattern for Test Vault Creation +A pytest fixture function that builds a complete vault directory tree under `tmp_path` with paperforge.json, directory structure, library records, and BBT export files. Every test gets an isolated, reproducible vault. + +**When to use:** When multiple tests need a realistic project structure but must never touch the real user data. The fixture factory guarantees isolation and reproducibility. + +**Source:** 05-02-PLAN.md, 05-02-SUMMARY.md + +--- + +### P2: Mocked HTTP via `unittest.mock.patch` +Use `unittest.mock.patch` to replace `requests.get` and `requests.post` with MagicMock objects that return controlled responses. Side effects raise exceptions for error-path testing. + +**When to use:** Testing code paths that make HTTP calls where you need to control response content, status codes, and error conditions without a real network. + +**Source:** 05-02-PLAN.md, 05-02-SUMMARY.md + +--- + +### P3: Side-Effect Factory Pattern for Mutable Return Values +When a mocked function returns a mutable object (like a dict) that the caller mutates, use `side_effect` with a factory function (not `return_value` with a single object) to produce a fresh object on each call. + +**When to use:** When a mocked function is called multiple times and the caller mutates the return value. A single shared object would accumulate state across calls. + +**Source:** 05-01-SUMMARY.md + +--- + +### P4: Real Minimal PDF Creation for File I/O Tests +Create a real (minimal) PDF file on disk using `b"%PDF-1.4\n..."` bytes for tests that need actual PDF file I/O, as opposed to mocking the file existence check. + +**When to use:** When testing file operations where you need actual file I/O behavior (open, read, stat) rather than just existence checks. A bytes-based stub file avoids external fixture dependencies. + +**Source:** 05-02-SUMMARY.md + +--- + +## Surprises + +### S1: 7 Attempts to Fix `test_missing_token_blocks_job` +The blocked state test for missing tokens required 7 implementation attempts because the standard `patch.dict(os.environ, {}, clear=True)` approach was ineffective — the registry token is always present in the test environment. + +**Impact:** Delayed completion of Plan 05-01 Task 1 by approximately 15 minutes. Ultimately fixed by using HTTPError 401 as a side_effect to trigger the `classify_error` path directly. This is now a documented testing pattern (see L1 above). + +**Source:** 05-01-SUMMARY.md + +--- + +### S2: Zero Deviations in Plan 05-02 +The smoke test plan (05-02) was executed exactly as written with no deviations, no issues encountered, and no auto-fixes needed. + +**Impact:** All 13 smoke tests passed on first implementation attempt. The fixture factory pattern and tmp_path isolation worked exactly as designed. + +**Source:** 05-02-SUMMARY.md + +--- + +### S3: Existing Base View Tests Already Covered Custom Paths +During Plan 05-01 Task 2 (base rendering with custom paths), it was discovered that the existing `test_base_views.py` (12 tests) and `test_base_preservation.py` (9 tests) already adequately covered custom path rendering via `substitute_config_placeholders` tests and force/non-force merge behavior. + +**Impact:** No new tests were needed for Task 2 — the existing 21 tests provided sufficient coverage. Saved approximately 20 minutes of test authoring. + +**Source:** 05-01-SUMMARY.md diff --git a/.planning/phases/06-setup-cli-diagnostics-consistency/06-LEARNINGS.md b/.planning/phases/06-setup-cli-diagnostics-consistency/06-LEARNINGS.md new file mode 100644 index 00000000..610b01a8 --- /dev/null +++ b/.planning/phases/06-setup-cli-diagnostics-consistency/06-LEARNINGS.md @@ -0,0 +1,124 @@ +--- +phase: 06 +phase_name: "Setup CLI Diagnostics Consistency" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 7 + lessons: 4 + patterns: 3 + surprises: 2 +missing_artifacts: + - "VERIFICATION.md" + - "UAT.md" +--- + +## Decisions + +### D-01: Use `ld_deep_script` as canonical field name +Consistently use `ld_deep_script` (not `literature_script`) in command documents to match `paperforge paths --json` output. + +**Rationale/Context:** Field name drift between docs and actual config output caused Agent confusion. D-02, D-03 from CONTEXT.md locked the name. +**Source:** 06-01-PLAN.md (Task 1) + +### D-02: `python -m paperforge` as documented fallback +When `paperforge` CLI command is not registered (e.g., user hasn't run `pip install`), `python -m paperforge` is the documented alternative. + +**Rationale/Context:** Provides a universal fallback that doesn't require PYTHONPATH or pip installation. D-16, D-17 from CONTEXT.md. +**Source:** 06-01-PLAN.md (Task 2) + +### D-03: HTTP 405 "Method Not Allowed" gets distinct diagnostic message +Doctor L2 check distinguishes HTTP 405 from other HTTP errors with an actionable message about method mismatch (GET vs POST). + +**Rationale/Context:** Generic "HTTP error" message didn't help users fix 405 issues. D-10, D-11, D-12 from CONTEXT.md. +**Source:** 06-01-PLAN.md (Task 3) + +### D-04: Doctor validates all `*.json` exports, not only `library.json` +The doctor's export validation uses `glob("*.json")` instead of checking for a single hardcoded `library.json`. + +**Rationale/Context:** BBT can generate per-domain JSON files; requiring `library.json` specifically was too restrictive. D-07, D-08, D-09 from CONTEXT.md. +**Source:** 06-02-PLAN.md (Task 1) + +### D-05: Vault path prefilled from `--vault` CLI argument into wizard +`setup_wizard.py` passes the `--vault` command-line argument through to `VaultStep` to prefill the input widget. + +**Rationale/Context:** Users had to re-type the vault path in the wizard UI even when specified on the command line. D-13, D-14, D-15 from CONTEXT.md. +**Source:** 06-02-PLAN.md (Task 2) + +### D-06: Doctor uses `paperforge_paths()` resolver for path reporting +The doctor reports `worker_script` path via the same `paperforge_paths()` resolver used by `paperforge paths --json`. + +**Rationale/Context:** Hardcoded paths in doctor could drift from actual config. Using the central resolver ensures consistency. D-18, D-19 from CONTEXT.md. +**Source:** 06-02-PLAN.md (Task 3) + +### D-07: `PADDLEOCR_API_TOKEN` is canonical env var name +All three system components (setup wizard, worker, doctor) consistently use `PADDLEOCR_API_TOKEN` for the API token. + +**Rationale/Context:** Env var names had drift between components. D-04, D-05, D-06 from CONTEXT.md locked the canonical name. +**Source:** 06-02-PLAN.md (Task 3) + +--- + +## Lessons + +### L-01: Doctor was too restrictive on JSON export validation +The doctor's export check only validated `library.json`, but Better BibTeX can generate per-domain `*.json` files. This caused false "missing export" warnings when `library.json` didn't exist but other JSON files were present. + +**Rationale/Context:** Previous implementation assumed a single `library.json` was the only valid export format. BBT's per-domain export feature was missed. +**Source:** 06-02-PLAN.md (Task 1) / 06-02-SUMMARY.md (Task 1) + +### L-02: Vault path wasn't propagating through setup wizard +The `--vault` CLI argument was accepted but not actually passed to the InteractiveTextUI's VaultStep, forcing users to manually re-enter paths. + +**Rationale/Context:** `SetupWizardApp.__init__` stored the vault parameter but didn't forward it to VaultStep's compose method. +**Source:** 06-02-PLAN.md (Task 2) / 06-02-SUMMARY.md (Task 2) + +### L-03: Environment variable name drift across components +Different parts of the system used different names for the same API token env var (e.g., `TOKEN` vs `KEY` vs `PADDLEOCR_API_TOKEN`). + +**Rationale/Context:** Independent development without cross-component naming conventions caused inconsistency. +**Source:** 06-02-PLAN.md (Task 3) / 06-02-SUMMARY.md (Task 3) + +### L-04: ProgressBar "stall" was a terminal display issue, not a logic bug +The wizard's ProgressBar appeared to "stall" because of terminal rendering behavior, not because the underlying step transitions were missing. + +**Rationale/Context:** Confirmed in D-15 that ProgressBar exists and advances correctly; the visual stall is a display artifact. +**Source:** 06-02-PLAN.md (Task 2, D-15) + +--- + +## Patterns + +### P-01: Independent doc fixes parallelized across waves +Multiple small doc fixes (field name, fallback command) are grouped into an independent wave that doesn't block other work. + +**When to use:** When tasks modify different files with no shared dependencies, execute them in parallel. +**Source:** 06-01-PLAN.md (overview: "independent doc fixes") + +### P-02: Glob-based validation over hardcoded filenames +Using `glob("*.json")` instead of `Path("library.json").exists()` future-proofs against changes in export format. + +**When to use:** When validating file existence in directories that may contain multiple valid files of a type. +**Source:** 06-02-PLAN.md (Task 1) / 06-02-SUMMARY.md (Task 1) + +### P-03: Centralized path resolver for cross-component consistency +`paperforge_paths()` serves as the single source of truth for script paths across doctor, CLI, and configuration. + +**When to use:** When multiple components need to reference the same filesystem paths. One resolver, one contract. +**Source:** 06-02-PLAN.md (Task 3) + +--- + +## Surprises + +### S-01: `literature_script` field name survived in docs after refactoring +Despite previous work, the incorrect field name `literature_script` was still present in `command/ld-deep.md` lines 170 and 194. + +**Impact:** Low — caused Agent command failures but no data loss. Fixed by replacing 2 occurrences. +**Source:** 06-01-PLAN.md (Task 1) + +### S-02: JSON export validation was checking only `library.json` +The doctor assumed `library.json` was the only valid BBT export, but BBT can generate per-domain JSON files. The single-file check was too restrictive. + +**Impact:** Medium — caused false positive "missing export" warnings for users using per-domain export configuration. +**Source:** 06-02-PLAN.md (Task 1) / 06-02-SUMMARY.md (Task 1) diff --git a/.planning/phases/07-zotero-pdf-metadata-state-repair/07-LEARNINGS.md b/.planning/phases/07-zotero-pdf-metadata-state-repair/07-LEARNINGS.md new file mode 100644 index 00000000..fc9826b4 --- /dev/null +++ b/.planning/phases/07-zotero-pdf-metadata-state-repair/07-LEARNINGS.md @@ -0,0 +1,124 @@ +--- +phase: 07 +phase_name: "Zotero PDF Metadata State Repair" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 5 + lessons: 5 + patterns: 3 + surprises: 3 +missing_artifacts: + - "VERIFICATION.md" + - "UAT.md" +--- + +## Decisions + +### D-01: `validate_ocr_meta()` must be called before using `ocr_status` in deep reading +`run_deep_reading()` was reading `ocr_status` directly from `meta.json` without validation. Changed to call `validate_ocr_meta()` which checks 7 conditions (file existence, size, page markers) before confirming status. + +**Rationale/Context:** `meta.json` with `ocr_status: done` but missing/corrupt files would falsely report papers as ready for deep reading. +**Source:** 07-PLAN.md (Task 2) + +### D-02: `done_incomplete` treated as blocked (not ready) +`validate_ocr_meta()` can return `done_incomplete`, which should be treated as blocked (needs re-OCR) rather than ready or waiting. + +**Rationale/Context:** Deep reading queue had a binary filter (`ocr_status == 'done'`) that didn't account for the `done_incomplete` state. +**Source:** 07-PLAN.md (Task 2) + +### D-03: Repair command defaults to dry-run with `--fix` flag +`paperforge repair` scans for state divergence by default and does nothing. `--fix` flag applies actual changes to divergent records. + +**Rationale/Context:** Destructive state changes should be opt-in. Users scan, review, then apply. +**Source:** 07-PLAN.md (Task 3, 4) + +### D-04: Three-way state comparison for detecting configuration drift +Repair compares three sources: library_record frontmatter, formal_note frontmatter, and meta.json (post-validation). + +**Rationale/Context:** State can diverge because different processes write to different files independently. Three-way comparison catches all inconsistency modes. +**Source:** 07-PLAN.md (Task 3) + +### D-05: BBT bare path normalization deferred +Normalizing bare `KEY/KEY.pdf` to `storage:KEY/KEY.pdf` in `load_export_rows()` was not implemented. The code change was deferred in favor of state repair work. + +**Rationale/Context:** `run_repair()` and OCR meta validation address active state corruption. BBT bare path handling has no concrete reproduction case. +**Source:** 07-SUMMARY.md (Task 1 — Deferred) + +--- + +## Lessons + +### L-01: Deep-reading bypassed OCR validation entirely +`run_deep_reading()` read `ocr_status` directly from `meta.json` without calling `validate_ocr_meta()`, even though that function existed and checked 7 conditions. + +**Rationale/Context:** The validation function was written but not wired into the queue logic. Two independently developed features weren't connected. +**Source:** 07-PLAN.md (Task 2) + +### L-02: Uninitialized variables in try/except blocks cause NameError +The `validated_error` variable was referenced outside a try block where it was assigned, causing a potential `NameError` if the assignment didn't execute. + +**Rationale/Context:** Fix by initializing `validated_status` and `validated_error` before the try block. +**Source:** 07-SUMMARY.md (Known Issues) + +### L-03: Dead code accumulates without detection +`domain_lookup` was discovered and removed during review — dead code that was previously unreachable. + +**Rationale/Context:** No automated dead code detection was in place. Discovered during manual review. +**Source:** 07-SUMMARY.md (Known Issues) + +### L-04: Edge cases in state comparison (None meta_ocr_status) +Case 3 (where `meta_ocr_status` is `None`) needed special handling added during implementation. The initial implementation assumed meta always had a valid status. + +**Rationale/Context:** Missing/incomplete meta.json leads to `None` status, which wasn't accounted for in comparison logic. +**Source:** 07-SUMMARY.md (Known Issues) + +### L-05: `O(n*m)` scan scales poorly with vault size +The `_resolve_formal_note_path()` function uses per-record `rglob`, which scales poorly with large vaults. + +**Rationale/Context:** Acceptable for current vault sizes but would become a bottleneck with hundreds of records. +**Source:** 07-SUMMARY.md (Known Issues) + +--- + +## Patterns + +### P-01: Three-way state comparison for configuration drift +Compare three data sources (library_record, formal_note, meta.json) to detect inconsistent states. + +**When to use:** When multiple independent processes write to different files and state must remain synchronized. +**Source:** 07-PLAN.md (Task 3) + +### P-02: Dry-run default with explicit fix flag +Commands that modify state do nothing by default (scan/display). Users must opt in with `--fix` to apply changes. + +**When to use:** Any command that could modify user data. Provides safety while allowing repair. +**Source:** 07-PLAN.md (Task 3, 4) + +### P-03: Test-first for defect fixes +Write a failing test that reproduces the bug, then implement the fix, then verify the test passes. + +**When to use:** For any bug fix where the expected behavior is clearly defined. Prevents regression and proves the fix works. +**Source:** 07-PLAN.md (all tasks) + +--- + +## Surprises + +### S-01: Deep-reading was reporting papers as ready when OCR files were missing/corrupt +`run_deep_reading()` trusted `meta.json` `ocr_status: done` without validating that the actual OCR files existed and were complete. This created a false-ready state. + +**Impact:** High — users would attempt deep reading on papers without valid OCR data. +**Source:** 07-PLAN.md (Task 2) / 07-SUMMARY.md + +### S-02: `done_incomplete` status existed but was unhandled in queue logic +`validate_ocr_meta()` could return `done_incomplete`, but `run_deep_reading()` only checked `ocr_status == 'done'`, causing `done_incomplete` papers to silently fall through. + +**Impact:** Medium — papers with incomplete OCR were neither ready nor visibly blocked, causing confusion. +**Source:** 07-PLAN.md (Task 2) + +### S-03: BBT bare path handling was lower priority than expected +The BBT bare `KEY/KEY.pdf` path normalization was deferred because no concrete reproduction case existed, despite being identified in the plan. + +**Impact:** Low — workaround exists (configure BBT with `storage:` prefix). Tracked for future phase. +**Source:** 07-SUMMARY.md (Task 1 — Deferred) diff --git a/.planning/phases/08-deep-helper-deployment/08-LEARNINGS.md b/.planning/phases/08-deep-helper-deployment/08-LEARNINGS.md new file mode 100644 index 00000000..a6f4e05b --- /dev/null +++ b/.planning/phases/08-deep-helper-deployment/08-LEARNINGS.md @@ -0,0 +1,129 @@ +--- +phase: 08 +phase_name: "Deep Helper Deployment" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 4 + lessons: 4 + patterns: 5 + surprises: 4 +missing_artifacts: + - "UAT.md" +--- + +## Decisions + +### D-01: Use `importlib.util.spec_from_file_location` with sys.modules pre-registration for Python 3.14 +When importing `ld_deep.py` from tests, pre-register the module in `sys.modules` before `exec_module` to work around Python 3.14 dataclass compatibility issues with `from __future__ import annotations`. + +**Rationale/Context:** Python 3.14 pre-release has a dataclass regression where `dataclasses` accesses `sys.modules[cls.__module__].__dict__` before the module is registered. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 2) + +### D-02: Generate deterministic OCR fixtures once and commit, never regenerate in CI +The `tests/sandbox/ocr-complete/TSTONE001/` fixtures (fulltext, figure-map, chart-type-map, meta) were generated once and committed to git. + +**Rationale/Context:** Deterministic fixtures ensure tests are repeatable. Regenerating in CI would introduce variability or require live API calls. +**Source:** 08-PLAN.md (Task 2) / 08-SUMMARY.md (Decisions Made) + +### D-03: Rollback deletes partial files and restores original note text +`prepare_deep_reading()` rollback tracks files written during the current run and, on failure, deletes created files and restores the original formal note from a saved copy. + +**Rationale/Context:** Not a full filesystem snapshot — only tracks artifacts created during this single prepare call. Sufficient for the partial-failure scenario. +**Source:** 08-PLAN.md (Task 5) / 08-SUMMARY.md (Decisions Made) + +### D-04: `_import_ld_deep()` helper for non-package imports +Created a helper function using `importlib.util.spec_from_file_location` to import `ld_deep.py` from outside the package tree, since `skills/` has no `__init__.py`. + +**Rationale/Context:** `from skills.literature_qa.scripts.ld_deep import ...` fails because `skills/` directory is not a Python package. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 3) + +--- + +## Lessons + +### L-01: Python 3.14 pre-release has a dataclass regression with `from __future__ import annotations` +`dataclasses.dataclass` accesses `sys.modules[cls.__module__].__dict__` before the module is registered in `sys.modules`, causing import failures when using `importlib.util.spec_from_file_location`. + +**Rationale/Context:** Only affects the pre-release Python 3.14.0. Workaround is to pre-register the module before `exec_module`. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 2) / 08-VERIFICATION.md + +### L-02: Doctor was checking directory existence, not actual importability +The doctor's "Agent script" check only verified that the script directory existed, not that the script could actually be imported and executed. + +**Rationale/Context:** Directory existence ≠ importability. A file can exist but fail to import due to syntax errors, missing dependencies, or module resolution issues. +**Source:** 08-PLAN.md (Task 1b) + +### L-03: `skills/` is not a Python package — direct import fails +Because `skills/` has no `__init__.py`, `from skills.literature_qa.scripts.ld_deep import ...` raises `ModuleNotFoundError`. + +**Rationale/Context:** The `skills/` directory is designed for deployment to Agent vaults, not as a Python package in the repo. An import helper is needed for tests. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 3) + +### L-04: zotero_key regex captured surrounding quotes causing queue filter mismatch +The regex extracting `zotero_key` from JSON captured quotation marks (`"TSTONE001"` instead of `TSTONE001`), causing the queue filter to not match records and OCR path lookup to fail. + +**Rationale/Context:** Fixed by adding `.strip('"').strip("'")` to the extracted key value. Found during test execution of `test_queue_shows_ready_paper`. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 1) + +--- + +## Patterns + +### P-01: Rollback-on-failure in prepare operations +Track files written during a prepare operation. On any exception, delete created files and restore original content from a saved copy. + +**When to use:** Any multi-step write operation where partial completion would leave the system in an inconsistent state. +**Source:** 08-PLAN.md (Task 5) / 08-SUMMARY.md + +### P-02: Deterministic fixtures committed to git +Generate test fixtures once via a script, verify correctness, and commit them. Never regenerate in CI. + +**When to use:** When tests depend on specific data that's expensive or non-deterministic to generate at test time. +**Source:** 08-PLAN.md (Task 2) / 08-SUMMARY.md + +### P-03: Regression test per reported issue +Each regression test is named with a regression ID and covers one specific issue from prior audit findings. + +**When to use:** When tracking regression from an audit or known-issue list. One test per issue = clear pass/fail per finding. +**Source:** 08-PLAN.md (Task 3d) / 08-SUMMARY.md (patterns-established) + +### P-04: Import helper for non-package modules +Use `importlib.util.spec_from_file_location` with a helper function to import modules that live outside the package tree. + +**When to use:** When code is deployed to a non-standard location (e.g., an Agent vault's skills directory) and tests need to import it. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 3) + +### P-05: Doc-as-executable validation +Extract command snippets from markdown documentation, execute them against a test fixture, and assert they run without error. + +**When to use:** When documentation contains runnable commands that must stay in sync with actual CLI behavior. +**Source:** 08-PLAN.md (Task 4) + +--- + +## Surprises + +### S-01: zotero_key quote stripping bug discovered during test execution +The regex capturing `zotero_key` from JSON included surrounding quotation marks. This wasn't caught by existing tests because no test exercised the queue with actual JSON-sourced keys. + +**Impact:** Medium — queue filter silently failed to match records, causing false "nothing ready" output. Caught and fixed during Phase 8 testing. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 1) + +### S-02: Python 3.14 dataclass import workaround needed +Python 3.14 pre-release broke the standard `importlib.util` pattern for importing modules with dataclasses. Required pre-registration in `sys.modules`. + +**Impact:** Low — workaround exists. Affects only pre-release Python 3.14. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 2) + +### S-03: Import from `skills/` fails because it's not a Python package +`skills/` has no `__init__.py`, so direct imports from it fail. Required creating an `_import_ld_deep()` helper. + +**Impact:** Medium — test imports would fail without the workaround. Not a production issue but a test infrastructure concern. +**Source:** 08-SUMMARY.md (Deviations — Auto-fixed Issue 3) + +### S-04: Test vault fixture cleanup caused transient state issues +Reusing the same test vault across tests caused state leaks (files from one test affecting another). Resolved by creating a fresh vault per test. + +**Impact:** Low — transient test failures during development. Resolved with per-test isolation. +**Source:** 08-SUMMARY.md (Issues Encountered) diff --git a/.planning/phases/09-command-unification/09-LEARNINGS.md b/.planning/phases/09-command-unification/09-LEARNINGS.md new file mode 100644 index 00000000..1bedde4a --- /dev/null +++ b/.planning/phases/09-command-unification/09-LEARNINGS.md @@ -0,0 +1,136 @@ +--- +phase: 09 +phase_name: "Command Unification" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 7 + lessons: 4 + patterns: 4 + surprises: 3 +missing_artifacts: + - "VERIFICATION.md" + - "UAT.md" +--- + +## Decisions + +### D-01: Aggressive migration — no aliases or deprecation for old Agent commands +Old commands (`/LD-*`, `/lp-*`) were completely removed with no aliases or deprecation period. Clean break for v1.2. + +**Rationale/Context:** Deprecation warnings add complexity without benefit in a CLI tool with few users. Cleaner to migrate aggressively with clear documentation. +**Source:** 09-SUMMARY.md (Key Decisions) + +### D-02: Unified `paperforge sync` runs both selection-sync and index-refresh +The new `sync` subcommand runs both phases by default. `--selection` and `--index` flags allow partial execution. + +**Rationale/Context:** Most users want to run both phases together. The two-step process was an implementation detail leaking into the user interface. +**Source:** 09-PLAN.md (Task 2) / 09-SUMMARY.md (Key Decisions) + +### D-03: Merged `paperforge ocr` combines run + doctor +The new `ocr` subcommand integrates both OCR execution and diagnostics. `--diagnose` flag runs standalone diagnostics. + +**Rationale/Context:** `ocr run` and `ocr doctor` were always used together in practice. Merging reduces command surface area. +**Source:** 09-PLAN.md (Task 2) / 09-SUMMARY.md (Key Decisions) + +### D-04: All CLI logic extracted into `paperforge/commands/` modules +Each subcommand has a dedicated module in `paperforge/commands/` exposing a `run(args_namespace)` interface. `cli.py` becomes a thin argparse wrapper. + +**Rationale/Context:** Shared logic between CLI and Agent layers requires importable modules, not inline argparse handlers. Enables command reuse. +**Source:** 09-PLAN.md (Task 1) / 09-SUMMARY.md (Key Decisions) + +### D-05: Python package unified under `paperforge` name +Renamed from `paperforge_lite` to `paperforge` across 98+ import statements and 377+ markdown references. + +**Rationale/Context:** The `_lite` suffix was a historical artifact. CLI command and Python package should share the same name. +**Source:** 09-PLAN.md (Task 0) / 09-SUMMARY.md + +### D-06: Old command references kept only in migration sections +AGENTS.md and other docs keep old command names (`selection-sync`, `/LD-*`) only in dedicated migration tables, not in primary documentation flow. + +**Rationale/Context:** Primary docs should reflect the current interface. Migration references help existing users but shouldn't clutter normal usage sections. +**Source:** 09-05-SUMMARY.md (Decisions Made) + +### D-07: Old CLI commands still function for backward compatibility +`cli.py` preserves backward-compatible aliases for old CLI commands (`selection-sync`, `index-refresh`, `ocr run`), even though docs only show new names. + +**Rationale/Context:** Prevents breaking existing user scripts and muscle memory. The aliases are not documented — users are encouraged to migrate. +**Source:** 09-05-SUMMARY.md (Decisions Made) + +--- + +## Lessons + +### L-01: Aggressive rename of 98+ Python imports is feasible when systematic +The package rename from `paperforge_lite` to `paperforge` touched 98+ import statements and 377+ markdown references, but was completed without introducing new failures. + +**Rationale/Context:** Systematic search-and-replace with automated verification (running tests, checking imports) makes large renames safe. +**Source:** 09-SUMMARY.md + +### L-02: Backward-compatible CLI aliases prevent breaking existing workflows +Old CLI commands (`selection-sync`, `index-refresh`, `ocr run`) were preserved as aliases, ensuring users' existing scripts and habits continue to work. + +**Rationale/Context:** Breaking changes should be opt-in. Backward-compatible aliases allow gradual migration without user disruption. +**Source:** 09-05-SUMMARY.md (Decisions Made) + +### L-03: Automated tests prevent regression of old command names in docs +`TestUnifiedCommandsInUserDocs` catches any accidental re-introduction of old command names in primary documentation sections. + +**Rationale/Context:** Without automation, docs can regress when someone copies old text or uses outdated references. +**Source:** 09-05-SUMMARY.md (Decisions Made) + +### L-04: Pre-existing test failures unrelated to command unification +Two test files (`test_pdf_resolver.py`, `test_base_preservation.py`, `test_base_views.py`) had pre-existing failures caused by `pipeline` module import issues, not by Phase 9 changes. + +**Rationale/Context:** Important to establish baseline test results before a refactoring phase so new breakage can be distinguished from pre-existing issues. +**Source:** 09-SUMMARY.md (Known Issues) + +--- + +## Patterns + +### P-01: Command module pattern with `run(args)` interface +Each submodule in `paperforge/commands/` exposes a uniform `run(args_namespace)` interface, registered in `__init__.py`. + +**When to use:** When building a CLI where commands should be independently importable and testable, and where multiple callers (CLI, Agent, tests) need access. +**Source:** 09-PLAN.md (Task 1) + +### P-02: Thin CLI wrapper pattern +`cli.py` is reduced to argparse dispatch only. All business logic lives in `paperforge/commands/` modules. + +**When to use:** When the CLI layer should be a thin adapter, not an application. Keeps command logic reusable across interfaces. +**Source:** 09-PLAN.md (Task 2) / 09-SUMMARY.md + +### P-03: Migration documentation pattern +Old command names appear only in dedicated migration sections (tables, sidebars), not in primary documentation flow. + +**When to use:** During any interface migration. Helps existing users transition while keeping new users focused on current syntax. +**Source:** 09-05-SUMMARY.md (Decisions Made) + +### P-04: Registry-based command discovery +`paperforge/commands/__init__.py` acts as a command registry, allowing `cli.py` to discover and dispatch to subcommands without hardcoded import chains. + +**When to use:** When the number of subcommands grows and you want a single point of registration rather than scattered imports. +**Source:** 09-PLAN.md (Task 1) + +--- + +## Surprises + +### S-01: Zero deviations from plan across all 6 tasks +Every task was executed exactly as written in the plan. No unexpected issues, no auto-fixes, no scope changes during Task 6 verification. + +**Impact:** Positive — demonstrates that the planning process has matured to high accuracy. +**Source:** 09-SUMMARY.md (Deviation from Plan) + +### S-02: 155 tests still pass despite 98+ Python import changes +A massive rename touching almost every Python file in the project didn't break a single passing test. + +**Impact:** Positive — systematic rename with verification works. Validates the testing infrastructure's coverage. +**Source:** 09-SUMMARY.md (Test Results) + +### S-03: Pre-existing `pipeline` module import errors in two test files +`test_base_preservation.py` and `test_base_views.py` import from `pipeline.worker.scripts.literature_pipeline` which doesn't exist in the current package structure. + +**Impact:** Low — pre-existing issue. Tests were likely created for a different module layout and never updated. +**Source:** 09-SUMMARY.md (Known Issues) diff --git a/.planning/phases/10-documentation-and-cohesion/10-LEARNINGS.md b/.planning/phases/10-documentation-and-cohesion/10-LEARNINGS.md new file mode 100644 index 00000000..6ce0c0eb --- /dev/null +++ b/.planning/phases/10-documentation-and-cohesion/10-LEARNINGS.md @@ -0,0 +1,119 @@ +--- +phase: 10 +phase_name: "Documentation and Cohesion" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 6 + lessons: 3 + patterns: 1 + surprises: 1 +missing_artifacts: + - "10-VERIFICATION.md" + - "10-UAT.md" +--- + +## Decisions + +### ADR records in ARCHITECTURE.md +10 Architecture Decision Records (ADR-001 through ADR-010) covering Phases 1-9 were documented in a single `docs/ARCHITECTURE.md` file, providing a unified reference for maintainers. + +**Rationale/Context:** Previous architectural decisions were scattered across phase context files. Consolidating them into one document with ADR format makes the codebase self-documenting for new contributors. + +**Source:** 10-PLAN.md (Task 1) + +--- + +### Unified command documentation template +All 5 `command/pf-*.md` files were standardized with a common structure: Purpose, CLI Equivalent, Prerequisites, Arguments, Example, Output, Error Handling, Platform Notes. + +**Rationale/Context:** Prior command docs had inconsistent structure. The unified template ensures every command doc covers the same categories of information, reducing ambiguity. + +**Source:** 10-PLAN.md (Task 4) + +--- + +### Agent-to-CLI mapping matrix in COMMANDS.md +A master command reference was created (`docs/COMMANDS.md`) mapping all 5 agent commands (`/pf-deep`, `/pf-paper`, `/pf-ocr`, `/pf-sync`, `/pf-status`) to their CLI equivalents with explicit requirements. + +**Rationale/Context:** Users and agents needed a quick way to understand which CLI command underlies each agent command and what prerequisites are needed. + +**Source:** 10-PLAN.md (Task 3) + +--- + +### Consistency audit as automated script +`scripts/consistency_audit.py` checks 4 hard constraints: no old command names, no `paperforge_lite` in Python code, no dead markdown links, and valid command doc structure. + +**Rationale/Context:** Manual consistency review is error-prone. An automated script can be run in CI to enforce hard constraints on every change. + +**Source:** 10-PLAN.md (Task 5) + +--- + +### Manual checklist for soft constraints +`docs/CONSISTENCY-CHECKLIST.md` covers terminology, branding, style, cross-references, version numbers, command examples, and agent command naming. + +**Rationale/Context:** Some consistency criteria (branding, terminology) cannot be automatically verified. A structured checklist provides a repeatable human review process. + +**Source:** 10-PLAN.md (Task 6) + +--- + +### Path normalization fix for bare BBT paths +Added `storage:` prefix normalization in `load_export_rows()` to handle BBT-exported bare `KEY/KEY.pdf` paths that were not being converted to the `storage:` format. + +**Rationale/Context:** Discovered during verification (Task 7) — the PDF resolver was failing on bare relative paths from BBT export. Fixing it ensured backward compatibility with existing exports. + +**Source:** 10-SUMMARY.md (Deviation Log, Item 2) + +--- + +## Lessons + +### Auto-fixed issues during verification catch latent bugs +Task 7 verification discovered and fixed 3 issues not in the plan: missing `pipeline/__init__.py` files (collection errors), bare path normalization (PDF resolver failure), and outdated test assertion (Phase 9 command unification changed "doctor" to "diagnose"). + +**Rationale/Context:** The verification step in Task 7 acted as a safety net, catching issues that were not identified during planning. This suggests that running the full test suite at phase boundaries is essential even for documentation-focused phases. + +**Source:** 10-SUMMARY.md (Deviation Log) + +--- + +### Baseline test count is not the final test count +Started with 155 passed / 2 skipped / 2 pre-existing failures as baseline. Verification revealed 6 new empty package dirs that caused collection errors. After fixes, final result: 178 passed / 2 skipped / 0 failed — a significant improvement from baseline. + +**Rationale/Context:** The verification process itself improved the codebase by discovering and fixing issues, demonstrating that phase-end verification adds value beyond confirmation. + +**Source:** 10-SUMMARY.md (Verification Results) + +--- + +### Consistency audit confirmed Phase 9 completeness +The audit found 0 occurrences of old command names and 0 references to `paperforge_lite` in Python code, confirming that Phase 9's command unification was fully and correctly executed. + +**Rationale/Context:** This cross-phase validation demonstrates the value of automated consistency checks for verifying that earlier phases' work was complete. + +**Source:** 10-SUMMARY.md (Verification Results) + +--- + +## Patterns + +### Verification-driven bug discovery +Running the test suite and consistency audit at phase completion (Task 7) catches both regression bugs and latent pre-existing issues. This doubles as both a quality gate and a cleanup mechanism. + +**Rationale/Context:** Even a documentation-focused phase (Phase 10) uncovered 3 functional bugs during verification. The pattern of "test at phase boundaries" should be standard for all phases. + +**Source:** 10-PLAN.md (Task 7), 10-SUMMARY.md (Deviation Log) + +--- + +## Surprises + +### Test suite discovered a real path normalization bug during a documentation phase +The pre-existing test suite caught a genuine functional bug (bare BBT paths not normalized to `storage:` prefix) during what was planned as a documentation-and-cohesion phase. This bug was causing PDF resolver failures in real usage. + +**Rationale/Context:** This was entirely unexpected — the phase was focused on documentation, not functional changes. The test suite's discovery of a live bug underscores the importance of running tests even in non-functional phases. + +**Source:** 10-SUMMARY.md (Deviation Log, Item 2) diff --git a/.planning/phases/11-zotero-path-normalization/11-LEARNINGS.md b/.planning/phases/11-zotero-path-normalization/11-LEARNINGS.md new file mode 100644 index 00000000..e00d0983 --- /dev/null +++ b/.planning/phases/11-zotero-path-normalization/11-LEARNINGS.md @@ -0,0 +1,172 @@ +--- +phase: 11 +phase_name: "Zotero Path Normalization" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 8 + lessons: 5 + patterns: 3 + surprises: 1 +missing_artifacts: + - "11-UAT.md" +--- + +## Decisions + +### Junctions resolved before computing relative paths (D-05) +`absolutize_vault_path()` resolves junctions before computing the vault-relative path, ensuring junction targets are used for wikilink computation rather than the junction point itself. + +**Rationale/Context:** Zotero data may live outside the vault. A junction at `vault/system/Zotero` pointing to `D:/Zotero` must be resolved first so the relative path correctly references the junction point, not the target. + +**Source:** 11-PLAN.md (Task 3, Decision D-05) + +--- + +### Doctor detects Zotero location and recommends junction (D-07) +`paperforge doctor` checks if Zotero is inside or outside the vault, and if outside, recommends the exact `mklink /J` command to create the required junction. + +**Rationale/Context:** Users often don't know where Zotero stores data or whether a junction is needed. Automated detection with copy-paste instructions removes guesswork from setup. + +**Source:** 11-PLAN.md (Task 5, Decision D-07) + +--- + +### Obsidian wikilink format with forward slashes (D-08) +All PDF paths are stored as `[[relative/path/to/file.pdf]]` with forward slashes, even on Windows. + +**Rationale/Context:** Obsidian wikilinks use forward slashes regardless of platform. Using `pathlib.Path.as_posix()` ensures cross-platform consistency. + +**Source:** 11-PLAN.md (Task 3, Decision D-08) + +--- + +### Hybrid main PDF identification strategy +Main PDF identified by: Priority 1 — attachment.title == "PDF" AND contentType == "application/pdf"; Priority 2 — largest PDF file; Priority 3 — first PDF in list. Remaining PDFs become supplementary. + +**Rationale/Context:** BBT exports don't always mark the main PDF. Some have title="PDF", some don't. The hybrid strategy maximizes correct identification across real-world export variations. + +**Source:** 11-PLAN.md (Task 02) + +--- + +### Three-format BBT path normalization +All 3 BBT export formats (absolute Windows paths, `storage:` prefix, bare relative) are first normalized to `storage:KEY/filename.pdf` intermediate format, then converted to wikilinks. + +**Rationale/Context:** Normalizing to an intermediate format decouples BBT parsing from Obsidian path resolution, making each concern independently testable. + +**Source:** 11-PLAN.md (Task 01) + +--- + +### path_error integration with repair and status +`paperforge repair --fix-paths` re-resolves path errors using current Zotero location. `paperforge status` reports path error count and suggests repair. + +**Rationale/Context:** Path errors can arise from Zotero data relocation or junction changes. Integrating with the repair system allows users to fix paths without manual editing of library-records. + +**Source:** 11-PLAN.md (Task 06) + +--- + +### New frontmatter fields for debugging and tracking +Library-records now include `bbt_path_raw`, `zotero_storage_key`, `attachment_count`, `supplementary` (wikilink list), and `path_error`. + +**Rationale/Context:** These fields enable debugging of path resolution issues, track attachment metadata, and support the repair workflow with structured error data. + +**Source:** 11-PLAN.md (Task 04) + +--- + +### ADR-011 for path normalization +The complete path normalization strategy (D-01 through D-08) was documented as ADR-011 in `docs/ARCHITECTURE.md`. + +**Rationale/Context:** Consistent with the architecture documentation pattern established in Phase 10, all major design decisions should be recorded as ADRs. + +**Source:** 11-VERIFICATION.md (Acceptance Criteria Verification) + +--- + +## Lessons + +### `as_posix()` is idiomatic over `replace("\\", "/")` +`pathlib.Path.as_posix()` converts backslashes to forward slashes and is more robust than string replacement. It was used for wikilink path formatting. + +**Rationale/Context:** The plan specified `replace("\\", "/")`, but `as_posix()` is the idiomatic Python approach. Functionally equivalent but cleaner. + +**Source:** 11-SUMMARY.md (Deviation 2) + +--- + +### Doctor functions belong in literature_pipeline.py, not ocr_diagnostics.py +The plan's acceptance criteria referenced `paperforge/ocr_diagnostics.py` for path resolution checks, but `run_doctor()` and all doctor infrastructure live in `pipeline/worker/scripts/literature_pipeline.py`. + +**Rationale/Context:** `ocr_diagnostics.py` is exclusively for OCR (PaddleOCR) tiered checks. Path resolution checks must be co-located with `run_doctor()` where the dispatch logic resides. + +**Source:** 11-SUMMARY.md (Deviation 3) + +--- + +### Test fixtures must match actual implementation behavior +`obsidian_wikilink_for_pdf()` joins `storage:` paths directly under `zotero_dir` (not `zotero_dir/storage/`). Test fixtures had to create PDFs at `zotero_dir/KEY/file.pdf` to match current behavior. + +**Rationale/Context:** The underlying path resolution behavior (whether `storage:` should include an implicit `storage/` segment) was deferred to Phase 12. Tests must faithfully reflect current implementation. + +**Source:** 11-VERIFICATION.md (Deviation 1) + +--- + +### Consistency audit catches dead links in documentation +Phase 11's consistency audit flagged links to non-existent `.planning/REQUIREMENTS-v1.2.md` and planning file markdown link syntax that resolved to dead links. + +**Rationale/Context:** Documentation accumulates stale cross-references over time. Automated dead-link detection is essential for maintaining documentation quality. + +**Source:** 11-VERIFICATION.md (Deviation 2) + +--- + +### 25 test methods needed for comprehensive path coverage +The test suite (`test_path_normalization.py`) required 25 test methods across 4 test classes to cover BBT path formats (8), main PDF identification (6), wikilink generation (6), and integration scenarios (5). + +**Rationale/Context:** Path normalization involves multiple input formats, edge cases, and output transformations. Adequate coverage required a dedicated test file with systematic test classes. + +**Source:** 11-VERIFICATION.md (Test Results) + +--- + +## Patterns + +### Combined tightly coupled tasks into single atomic commit +Tasks 03 and 04 both modified the same file (`literature_pipeline.py`) with interdependent changes. They were committed together as `adf349e` to maintain atomicity. + +**Rationale/Context:** When multiple tasks modify the same file with tight coupling, splitting into separate commits creates intermediate states that cannot be tested independently. + +**Source:** 11-SUMMARY.md (Deviation 1) + +--- + +### `storage:` prefix as intermediate representation +All 3 BBT path formats are normalized to `storage:KEY/file.pdf` as an intermediate step before final wikilink generation. + +**Rationale/Context:** This decouples BBT parsing from Obsidian path resolution, allowing each transformation to be tested independently and supporting future path format additions. + +**Source:** 11-PLAN.md (Task 01), 11-SUMMARY.md (Data Flow) + +--- + +### Acceptance criteria verified via grep patterns +All acceptance criteria were checked with exact `grep` patterns, ensuring traceable, machine-verifiable criteria. + +**Rationale/Context:** grep-verifiable criteria eliminate ambiguity in acceptance testing and enable automated verification during code review. + +**Source:** 11-SUMMARY.md (Acceptance Criteria Verification) + +--- + +## Surprises + +### Storage path resolution doesn't add implicit storage/ segment +`obsidian_wikilink_for_pdf()` resolves `storage:KEY/file.pdf` directly under `zotero_dir` (`zotero_dir/KEY/file.pdf`) rather than `zotero_dir/storage/KEY/file.pdf`. This differs from real Zotero directory structure. + +**Rationale/Context:** This behavior was inherited from the existing `pdf_resolver.py` implementation. Whether it's correct was deferred to Phase 12 architecture cleanup. Tests had to match current behavior despite it potentially being wrong. + +**Source:** 11-VERIFICATION.md (Deviation 1) diff --git a/.planning/phases/12-architecture-cleanup/12-LEARNINGS.md b/.planning/phases/12-architecture-cleanup/12-LEARNINGS.md new file mode 100644 index 00000000..9955a0f1 --- /dev/null +++ b/.planning/phases/12-architecture-cleanup/12-LEARNINGS.md @@ -0,0 +1,146 @@ +--- +phase: 12 +phase_name: "Architecture Cleanup" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 6 + lessons: 4 + patterns: 3 + surprises: 1 +missing_artifacts: + - "12-VERIFICATION.md" + - "12-UAT.md" +--- + +## Decisions + +### Extract 4041-line monolithic file into 7 focused worker modules +`paperforge/worker/` now contains 7 modules: sync.py, ocr.py, repair.py, status.py, deep_reading.py, update.py, base_views.py — each responsible for a single functional area. + +**Rationale/Context:** The original `literature_pipeline.py` (4041 lines) was a God module with no separation of concerns. Extraction improves maintainability, testability, and developer onboarding. + +**Source:** 12-SUMMARY.md (Files Created) + +--- + +### Relocate skills/ to paperforge/skills/ with backward-compatible fallback +`skills/literature-qa/` moved to `paperforge/skills/literature-qa/`. `setup_wizard.py` checks the new location first, then falls back to the old location during the transition period. + +**Rationale/Context:** Skills are part of the paperforge package, not an independent top-level directory. A fallback mechanism prevents breakage for users who haven't updated their installation. + +**Source:** 12-SUMMARY.md (Skills Migration, Decisions Made) + +--- + +### Duplicated utilities acceptable for test compatibility +Each worker module received a full copy of utility functions from the original monolithic file header during migration. + +**Rationale/Context:** Slightly wasteful, but ensures each module is self-contained during the migration without needing to rewrite all internal function calls. A planned refactoring pass in a future phase can extract shared utilities. + +**Source:** 12-SUMMARY.md (Decisions Made) + +--- + +### Module-reference imports for patchable cross-module calls +`ocr.py` uses `from paperforge.worker import sync as _sync` with `_sync.run_selection_sync()` instead of direct `from paperforge.worker.sync import run_selection_sync`. + +**Rationale/Context:** Direct imports create module-level references that are not affected by monkey-patching. Module-reference imports allow tests to patch `paperforge.worker.sync.run_selection_sync` and have the patch apply to calls within `ocr.py`. + +**Source:** 12-SUMMARY.md (Decisions Made) + +--- + +### Function-level imports to avoid circular imports +`sync.py` uses function-level imports inside `run_selection_sync` and `run_index_refresh` for cross-module dependencies (`ensure_base_views` from `base_views.py`, `validate_ocr_meta` from `ocr.py`). + +**Rationale/Context:** `sync.py` and `ocr.py` have mutual dependencies. Top-level imports in both directions would create circular import errors at module load time. Function-level imports defer the resolution until invocation. + +**Source:** 12-SUMMARY.md (Deviation 4) + +--- + +### Delete old directories only after test confirmation +`pipeline/` and `skills/` directories were removed only after all 203 tests passed with zero failures. + +**Rationale/Context:** Premature deletion could have hidden issues. Keeping old directories until test verification provides a rollback path and ensures the new module structure is fully functional. + +**Source:** 12-SUMMARY.md (Decisions Made) + +--- + +## Lessons + +### OCR test patches must target the correct module namespace +After extracting `run_ocr` to `paperforge/worker/ocr.py`, tests that patched `paperforge.worker.sync.write_json` no longer caught writes because `ocr.py` had its own copy of the utility function. + +**Rationale/Context:** Module extraction changes function ownership. Tests must update their patch targets to reflect the new module structure. This was discovered only when tests failed after the migration. + +**Source:** 12-SUMMARY.md (Deviation 1) + +--- + +### Direct imports break mock patching for cross-module calls +`run_ocr` called `run_selection_sync` via `from paperforge.worker.sync import run_selection_sync`, creating a module-level reference that was not affected by patches on `sync.run_selection_sync`. + +**Rationale/Context:** Python's `from X import Y` creates a new reference in the importing module's namespace. `unittest.mock.patch` targets are ineffective on these direct imports. The fix was to use module-reference imports (`import sync as _sync` + `_sync.run_selection_sync()`). + +**Source:** 12-SUMMARY.md (Deviation 2) + +--- + +### Dynamic module loading tests break when source files are deleted +`test_legacy_worker_compat.py` dynamically loaded the now-deleted `literature_pipeline.py` at import time, causing the test module itself to fail before any test could execute. + +**Rationale/Context:** Tests that dynamically load source files are tightly coupled to file paths. When those files are moved or deleted, the test module cannot even be imported. Tests should import from the package interface, not from specific file paths. + +**Source:** 12-SUMMARY.md (Deviation 3) + +--- + +### 203 tests passed after full migration +The complete architecture cleanup resulted in 203 passed, 0 failed, 2 skipped — meeting the target of 203+ passed. + +**Rationale/Context:** This validates that the extraction was functionally correct. All existing behavior was preserved despite the significant structural reorganization. + +**Source:** 12-SUMMARY.md (Test Results) + +--- + +## Patterns + +### Wave strategy with per-wave verification +Each wave (Package Structure + Sync/OCR, Repair/Status/Deep, Base Views + Skills, Import Cleanup, Docs) was verified individually before proceeding to the next, preventing regression accumulation. + +**Rationale/Context:** Architecture cleanup is inherently risky (many interdependent changes). Verifying each wave independently ensures problems are caught early and the rollback scope is minimized. + +**Source:** 12-PLAN.md (Wave Strategy) + +--- + +### Backward-compatible fallback in setup_wizard +The setup wizard checks the new directory location first, then falls back to the old location for the transition period. + +**Rationale/Context:** Users may have outdated installations or manual configurations. A graceful fallback prevents breakage during the migration window. + +**Source:** 12-SUMMARY.md (Decisions Made) + +--- + +### git mv for preserving file history +The plan explicitly recommended using `git mv` for file moves to preserve git history across the extraction. + +**Rationale/Context:** Without `git mv`, renamed files lose their commit history, making future `git blame` investigations harder. Preserving history aids debugging and attribution. + +**Source:** 12-PLAN.md (Risk Mitigation) + +--- + +## Surprises + +### Circular import issues between sync.py and ocr.py +`sym.py` needs functions from `base_views.py` and `ocr.py`, while `ocr.py` also needs `run_selection_sync` from `sync.py`. This mutual dependency created circular import errors that were not anticipated. + +**Rationale/Context:** In a monolithic 4041-line file, circular imports don't exist because everything is in the same module. Splitting by functional area revealed hidden coupling between sync and OCR logic that required function-level imports to break the cycle. + +**Source:** 12-SUMMARY.md (Deviation 4) diff --git a/.planning/phases/13-logging-foundation/13-LEARNINGS.md b/.planning/phases/13-logging-foundation/13-LEARNINGS.md new file mode 100644 index 00000000..f41a8739 --- /dev/null +++ b/.planning/phases/13-logging-foundation/13-LEARNINGS.md @@ -0,0 +1,217 @@ +--- +phase: 13 +phase_name: "Logging Foundation" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 11 + lessons: 5 + patterns: 5 + surprises: 1 +missing_artifacts: + - "13-UAT.md" +--- + +## Decisions + +### Single configure_logging(verbose) entry point (D-02) +`paperforge/logging_config.py` exports a single `configure_logging(verbose: bool = False)` function as the only entry point for all logging configuration. + +**Rationale/Context:** Prevents scattered `basicConfig()` calls across modules. A single entry point ensures consistent log behavior across all commands. + +**Source:** 13-01-PLAN.md (Decision D-02) + +--- + +### Programmatic setup over dictConfig (D-03) +Uses `logger.setLevel()` + `handler.setLevel()` programmatic setup instead of `dictConfig` or `fileConfig`. + +**Rationale/Context:** Simpler implementation, no external config files needed, and avoids the complexity of `dictConfig` for a straightforward logging setup. + +**Source:** 13-01-PLAN.md (Decision D-03) + +--- + +### --verbose maps to DEBUG level (D-05) +The `--verbose` / `-v` CLI flag forces `logging.DEBUG` level regardless of the `PAPERFORGE_LOG_LEVEL` environment variable. + +**Rationale/Context:** Users who explicitly pass `--verbose` want maximum detail, overriding any environment-level configuration. This provides a reliable way to get debug output on demand. + +**Source:** 13-01-PLAN.md (Decision D-05) + +--- + +### PAPERFORGE_LOG_LEVEL env var with INFO default (D-06) +Environment variable `PAPERFORGE_LOG_LEVEL` controls default log level. Accepted values: `DEBUG`, `INFO`, `WARNING`, `ERROR`. Default: `INFO`. + +**Rationale/Context:** Environment-level control allows system administrators and CI scripts to set log verbosity without modifying CLI invocations. + +**Source:** 13-01-PLAN.md (Decision D-06) + +--- + +### Invalid env var falls back to WARNING (D-09) +Invalid `PAPERFORGE_LOG_LEVEL` values (e.g., `banana`) silently fall back to `WARNING`. + +**Rationale/Context:** A typo in an environment variable should not crash the application. Falling back to a safe, less verbose level is better than failing silently at INFO or raising an error. + +**Source:** 13-01-PLAN.md (Decision D-09) + +--- + +### Idempotency guard prevents double configuration (D-10) +`if logger.handlers: return` guard ensures `configure_logging()` is safe to call multiple times. + +**Rationale/Context:** Early-boot code may call `logging.basicConfig()` or `configure_logging()` before the CLI's `main()` runs. The idempotency guard prevents duplicate handlers and duplicate log lines. + +**Source:** 13-01-PLAN.md (Decision D-10) + +--- + +### StreamHandler targets stderr +The logging StreamHandler writes to `sys.stderr`, not `sys.stdout`. + +**Rationale/Context:** Diagnostic output must be separated from user-facing output (OBS-02). Stdout is reserved for `print()` calls that produce command results; stderr carries log messages. This ensures piped commands and redirected output function correctly. + +**Source:** 13-01-PLAN.md (Implementation notes) + +--- + +### Formatter: LEVEL:name:message +Log format is `LEVEL:name:message` (e.g., `INFO:paperforge.worker.sync:Starting sync`). + +**Rationale/Context:** Compact format is grep-friendly. No timestamps because users see real-time terminal output. The hierarchical logger name (`paperforge.worker.sync`) enables targeted filtering. + +**Source:** 13-01-PLAN.md (Implementation notes) + +--- + +### printf-style format strings for all logger calls +All migrated logger calls use `%s`, `%d` formatting instead of f-strings. + +**Rationale/Context:** printf-style formatting enables lazy evaluation — if the log level suppresses a message, string formatting is never executed. This is a performance best practice for logging. + +**Source:** 13-02-SUMMARY.md (Decisions Made) + +--- + +### verbose=getattr(args, "verbose", False) as safe extraction pattern +All command modules use `getattr(args, "verbose", False)` to extract the verbose flag from argparse namespace. + +**Rationale/Context:** Handles the case where `args` might not have a `verbose` attribute (e.g., when command modules are called outside CLI dispatch in tests or offline contexts). The default `False` ensures backward compatibility. + +**Source:** 13-03-SUMMARY.md (Decisions Made) + +--- + +### Contract-first verbose parameter in worker functions +Worker functions accept `verbose: bool = False` parameter even when not yet used internally. + +**Rationale/Context:** The parameter serves as a contract between command and worker layers. Adding it upfront (before internal implementation) prevents cascading signature changes across all call sites later. + +**Source:** 13-03-SUMMARY.md (Decisions Made) + +--- + +## Lessons + +### argparse does NOT inherit root parser flags to subparsers +The plan assumed `deep-reading -v` would continue to work after moving `--verbose` to the root parser. This is incorrect — argparse only recognizes root-level flags when they appear BEFORE the subcommand name. `paperforge deep-reading -v` fails; `paperforge -v deep-reading` works. + +**Rationale/Context:** This is a fundamental argparse design limitation. The fix was to keep `--verbose` on the root parser only, document the syntax change, and accept the backward compatibility break. + +**Source:** 13-01-SUMMARY.md (Deviation 1) + +--- + +### paperforge/update.py (standalone) does not exist +The plan listed `paperforge/update.py` as a target file, but only `paperforge/worker/update.py` exists on disk. + +**Rationale/Context:** The plan incorrectly referenced a standalone update module that was never created. The actual update logic lives in the worker module hierarchy. This was discovered during execution and the non-existent file was skipped. + +**Source:** 13-02-SUMMARY.md (Deviation 1) + +--- + +### Removing _color() function breaks input() prompt +When `_log()` and `_color()` functions were removed from `worker/update.py`, an `input(_color("确认更新..."))` call remained, referencing the now-deleted `_color()`. + +**Rationale/Context:** The code review missed that `_color()` was used not only for logging but also for user prompts. The fix was to replace the colored prompt with plaintext `input("...")`. + +**Source:** 13-02-SUMMARY.md (Deviation 2) + +--- + +### 12/12 truths verified across all 3 sub-plans +The final verification confirmed all 12 observable truths from the plan's must-haves, 5/5 ROADMAP success criteria, and all key links as wired. + +**Rationale/Context:** Rigorous verification ensures no gaps. The structured must-haves/artifacts/key-links pattern from the plan translated directly into verifiable criteria. + +**Source:** 13-VERIFICATION.md (Must-Haves Truths Verification) + +--- + +### 50+ diagnostic print() calls migrated +Across repair.py (10), commands/ocr.py (4), and update.py (36+), the migration replaced scattered diagnostic `print()` calls with structured logging. + +**Rationale/Context:** This volume of print-to-logger migration demonstrates the extent of ad-hoc diagnostic output that had accumulated in the codebase. A systematic migration was needed, not a targeted cleanup. + +**Source:** 13-VERIFICATION.md (Gaps Summary) + +--- + +## Patterns + +### Root-level -v/--verbose for all commands +A single global `--verbose` flag on the root parser is inherited by all subcommands via argparse, eliminating per-subcommand flag duplication. + +**Rationale/Context:** Centralizing the flag simplifies argument parsing and ensures consistent behavior across all commands. Users must place `-v` before the subcommand. + +**Source:** 13-01-SUMMARY.md (Patterns Established) + +--- + +### configure_logging() as single entry point +Never call `logging.basicConfig()` or `dictConfig()` directly. Always use `configure_logging()` from `paperforge.logging_config`. + +**Rationale/Context:** Scattered logging configuration leads to inconsistent behavior (different formats, different levels, duplicate handlers). A single call point ensures determinism. + +**Source:** 13-01-SUMMARY.md (Patterns Established) + +--- + +### stdout reserved for user-facing output; stderr for diagnostics +`print()` calls that produce command results go to stdout. All diagnostic/trace/error output goes through logging to stderr. + +**Rationale/Context:** (OBS-02) Separating output streams ensures piped commands receive clean data on stdout while diagnostic messages are visible on stderr without polluting pipeline data. + +**Source:** 13-01-SUMMARY.md (Patterns Established), 13-VERIFICATION.md (Goal Achievement) + +--- + +### verbose=getattr(args, "verbose", False) standard wiring pattern +All command modules use this exact pattern to extract the verbose flag from argparse and pass it to worker functions. + +**Rationale/Context:** Standardized pattern is reviewable, testable, and safe for CLI and non-CLI invocation contexts. + +**Source:** 13-03-SUMMARY.md (Patterns Established) + +--- + +### def foo(vault, verbose=False) standard worker signature +All worker functions that accept verbose follow the same signature pattern: `def function_name(vault, verbose: bool = False) -> int:`. + +**Rationale/Context:** Consistent signatures across all worker modules reduce cognitive load for developers and enable straightforward automated tooling (e.g., signature introspection for documentation generation). + +**Source:** 13-03-SUMMARY.md (Patterns Established) + +--- + +## Surprises + +### Argparse root-to-subparser flag inheritance is one-directional +The plan's assumption that moving `--verbose` to the root parser would be transparent was incorrect. argparse does not pass root-level flags to subparsers when the flag appears after the subcommand name. This means `paperforge deep-reading -v` (which worked in v1.3) no longer works. Users must use `paperforge -v deep-reading`. + +**Rationale/Context:** This is a known argparse design choice, not a bug. Argparse parses arguments positionally — once the subcommand is consumed, the remaining arguments only match that subparser's definition. The fix required accepting a backward compatibility break, which was a necessary trade-off for the unified flag approach. + +**Source:** 13-01-SUMMARY.md (Deviation), 13-VERIFICATION.md (Deviations) diff --git a/.planning/phases/14-shared-utilities-extraction/14-00-PLAN.md b/.planning/phases/14-shared-utilities-extraction/14-00-PLAN.md new file mode 100644 index 00000000..cfe42a1c --- /dev/null +++ b/.planning/phases/14-shared-utilities-extraction/14-00-PLAN.md @@ -0,0 +1,9 @@ +--- +phase: 14 +title: Shared Utilities Extraction +status: complete +--- + +This phase was absorbed into later phases. The target outcome (`_utils.py` leaf module) was delivered through phases 20+. + +**Outcome:** Achieved via incremental work across Phases 15-25. No standalone phase execution required. diff --git a/.planning/phases/14-shared-utilities-extraction/14-00-SUMMARY.md b/.planning/phases/14-shared-utilities-extraction/14-00-SUMMARY.md new file mode 100644 index 00000000..12afc9c0 --- /dev/null +++ b/.planning/phases/14-shared-utilities-extraction/14-00-SUMMARY.md @@ -0,0 +1,9 @@ +--- +phase: 14 +title: Shared Utilities Extraction +plan: 14-00-PLAN.md +status: complete +actual_effort: 0 (absorbed into phases 15-25) +--- + +Shared utilities extraction was completed incrementally across subsequent phases. The `_utils.py` leaf module now serves as the single source for common utility functions used by all workers. diff --git a/.planning/phases/14-shared-utilities-extraction/14-CONTEXT.md b/.planning/phases/14-shared-utilities-extraction/14-CONTEXT.md index 3ee4cf9b..5bc53181 100644 --- a/.planning/phases/14-shared-utilities-extraction/14-CONTEXT.md +++ b/.planning/phases/14-shared-utilities-extraction/14-CONTEXT.md @@ -1,7 +1,7 @@ # Phase 14: Shared Utilities Extraction - Context **Gathered:** 2026-04-27 -**Status:** Ready for planning +**Status:** Complete (absorbed into phases 20+) ## Phase Boundary diff --git a/.planning/phases/15-deep-reading-queue-merge/15-LEARNINGS.md b/.planning/phases/15-deep-reading-queue-merge/15-LEARNINGS.md new file mode 100644 index 00000000..b33e5662 --- /dev/null +++ b/.planning/phases/15-deep-reading-queue-merge/15-LEARNINGS.md @@ -0,0 +1,114 @@ +--- +phase: 15 +phase_name: "Deep Reading Queue Merge" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 6 + lessons: 2 + patterns: 3 + surprises: 0 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +## Decisions + +### scan_library_records() returns ALL analyze=true records regardless of deep_reading_status + +The canonical function does not filter by deep_reading_status — it is the caller's responsibility to apply additional filters. This keeps the function pure (data acquisition only, no business logic). + +**Source:** 15-01-PLAN.md (task 1, line 136-147), 15-01-SUMMARY.md (Decisions Made) + +--- + +### scan_library_records() does NOT sort output + +Sorting is deferred to callers (deep_reading.py and ld_deep.py each have different sort requirements). The canonical function returns records in filesystem order. + +**Source:** 15-01-PLAN.md (task 1, line 167), 15-01-SUMMARY.md (Decisions Made) + +--- + +### ld_deep.py uses module-level direct import from _utils.py + +Since _utils.py is a leaf module (no intra-project dependencies), module-level imports from it are safe — no circular dependency risk. + +**Source:** 15-01-PLAN.md (task 2, lines 267-272), 15-01-SUMMARY.md (Decisions Made) + +--- + +### deep_reading.py retains status sync + categorization + report generation + +The refactored run_deep_reading() still checks has_deep_reading_content() against actual note content and syncs frontmatter before building the queue. Only the inline scanning loop was replaced. + +**Source:** 15-01-PLAN.md (task 2, lines 208-212), 15-01-SUMMARY.md (Decisions Made) + +--- + +### OCR status lookup simplified to direct meta.json read via read_json() + +Uses the same approach that ld_deep.py previously used with _read_json — direct read of meta.json and extraction of ocr_status field. + +**Source:** 15-01-PLAN.md (task 1, line 163), 15-01-SUMMARY.md (Decisions Made) + +--- + +### Regex patterns preserved from ld_deep.py for output consistency + +Canonical function uses identical frontmatter extraction regex patterns from ld_deep.py (e.g., `r'^analyze:\s*(true|false)$'`) to guarantee behavioral equivalence. + +**Source:** 15-01-PLAN.md (task 1, lines 158-162), 15-01-SUMMARY.md (Decisions Made) + +--- + +## Lessons + +### Module-level direct import is safe when target is a leaf module + +_utils.py is a leaf module (stdlib + paperforge.config only, no paperforge.worker.* imports). This makes module-level from _utils import ... safe in any consumer without circular import risk. + +**Source:** 15-01-PLAN.md (task 2), 15-01-SUMMARY.md (Decisions Made) + +--- + +### Pure data acquisition functions simplify testing and reasoning + +scan_library_records() has no side effects and no categorization logic. This separation of concerns makes it testable in isolation and predictable for all callers. + +**Source:** 15-01-PLAN.md (task 1), 15-01-SUMMARY.md + +--- + +## Patterns + +### Canonical Data Acquisition Function + +A single function in _utils.py that acquires data from library records with pure semantics (no side effects, no filtering, no sorting). All consumers delegate to it. + +**Source:** 15-01-PLAN.md (lines 57-64), 15-01-SUMMARY.md (patterns-established) + +--- + +### Thin-Wrapper Delegation + +Consumer modules wrap the canonical function with their own caller-specific filter and sort logic. The wrapper has minimal code (~14 lines) and delegates real work to the shared function. + +**Source:** 15-01-PLAN.md (task 2, lines 276-292), 15-01-SUMMARY.md (patterns-established) + +--- + +### Re-Export with Backward Compatibility Comment + +When a function moves from a worker module to _utils.py, the original location gets a short re-export comment (`# Re-exported from _utils.py for backward compatibility`) so existing importers continue working. + +**Source:** 15-01-PLAN.md (task 2, lines 199-201), 15-01-SUMMARY.md + +--- + +## Surprises + +None documented. Plan executed exactly as written with zero deviations. + +**Source:** 15-01-SUMMARY.md (Deviations from Plan) diff --git a/.planning/phases/16-retry-progress-bars/16-LEARNINGS.md b/.planning/phases/16-retry-progress-bars/16-LEARNINGS.md new file mode 100644 index 00000000..3e1869fa --- /dev/null +++ b/.planning/phases/16-retry-progress-bars/16-LEARNINGS.md @@ -0,0 +1,156 @@ +--- +phase: 16 +phase_name: "Retry Progress Bars" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 9 + lessons: 3 + patterns: 4 + surprises: 0 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +## Decisions + +### _retry.py and _progress.py follow leaf module pattern (no intra-project imports) + +Both modules import only from stdlib and external dependencies (tenacity, tqdm). No imports from paperforge.* modules, ensuring zero circular dependency risk. + +**Source:** 16-01-PLAN.md (interfaces), 16-01-SUMMARY.md + +--- + +### configure_retry() reads env vars with sensible defaults + +PAPERFORGE_RETRY_MAX defaults to 5; PAPERFORGE_RETRY_BACKOFF defaults to 2.0. Returns a tenacity retry decorator with exponential backoff, jitter, and retryable exception classification (ConnectionError, Timeout, HTTP 429/503). + +**Source:** 16-01-PLAN.md (task 2, lines 77-88), 16-01-SUMMARY.md + +--- + +### retry_with_meta writes retry metadata to meta.json on each attempt + +On each retry, writes retry_count, last_error, last_attempt_at to meta.json. On exhaustion, re-raises for caller to handle — does not catch internally. + +**Source:** 16-01-PLAN.md (task 2, lines 89-94), 16-01-SUMMARY.md + +--- + +### progress_bar wraps tqdm with stderr output per D-10 + +All tqdm output goes to sys.stderr so stdout remains clean for piped data. Wraps with mininterval=1.0 and compact bar format. + +**Source:** 16-01-PLAN.md (task 3, lines 203-213), 16-01-SUMMARY.md + +--- + +### --no-progress is a root-level global flag + +Added to the root argument parser (same level as --verbose), not any sub-parser. Accessible across all commands as args.no_progress. + +**Source:** 16-02-PLAN.md (task 1, lines 92-103), 16-02-SUMMARY.md + +--- + +### Zombie job detection threshold defaults to 30 minutes + +Jobs with ocr_status 'queued' or 'running' older than PAPERFORGE_ZOMBIE_TIMEOUT_MINUTES (default 30) are reset to 'pending' at run_ocr() startup. + +**Source:** 16-02-PLAN.md (task 2, lines 144-158), 16-02-SUMMARY.md + +--- + +### Poll failures must not abort batch (batch resilience per D-09) + +Previously uncaught poll raise_for_status() would crash the entire batch. Now wrapped in try/except with retry, continuing to next item on exhaustion. + +**Source:** 16-02-PLAN.md (task 2, lines 191-211), 16-02-SUMMARY.md + +--- + +### Upload and poll refactored into inner functions for clean retry wrapping + +_do_upload() and _do_poll() inner functions capture the actual API call, allowing retry_with_meta() to wrap them without restructuring the large run_ocr() loop. + +**Source:** 16-02-PLAN.md (task 2, lines 165-210), 16-02-SUMMARY.md + +--- + +### ensure_ocr_meta() extended with retry metadata defaults + +Three new setdefault lines: retry_count=0, last_error=None, last_attempt_at=None. This ensures every meta.json has the retry tracking fields even before any retry occurs. + +**Source:** 16-02-PLAN.md (task 2, lines 213-219), 16-02-SUMMARY.md + +--- + +## Lessons + +### Refactoring upload/poll into inner functions enables clean retry wrapping + +By extracting API call logic into small inner functions, retry_with_meta() can wrap them transparently without restructuring the entire run_ocr() loop or duplicating exception handling. + +**Context:** The original upload code was inline inside a large try/except block. Extracting _do_upload() and _do_poll() allowed retry_with_meta() to wrap the API call itself while the existing exception handler continued serving as the catch-all after exhaustion. + +**Source:** 16-02-PLAN.md (task 2), 16-02-SUMMARY.md + +--- + +### Zombie reset at startup handles stale processing states gracefully + +By scanning meta.json files for stale 'queued'/'running' states at run_ocr() start, zombie jobs from crashes or timeouts are automatically reset without manual intervention. + +**Source:** 16-02-PLAN.md (task 2), 16-02-SUMMARY.md + +--- + +### Existing exception handlers naturally serve as batch resilience catch-all + +After retry_with_meta() exhausts its attempts and re-raises, the existing except Exception block already handled classification, meta.json update, and continue. No additional structural changes were needed. + +**Source:** 16-02-PLAN.md (task 2, lines 248-251), 16-02-SUMMARY.md + +--- + +## Patterns + +### Leaf Module for Infrastructure + +New infrastructure modules (_retry.py, _progress.py) follow the existing _utils.py leaf module pattern: no intra-project imports, single purpose, stdout/stdlib-only dependencies. + +**Source:** 16-01-PLAN.md (tasks 2, 3), 16-01-SUMMARY.md + +--- + +### Env-Var-Based Configuration with Defaults + +Retry behavior (max attempts, backoff multiplier) and zombie timeout configured via environment variables with sensible defaults, matching the existing PAPERFORGE_LOG_LEVEL pattern. + +**Source:** 16-01-PLAN.md (task 2), 16-02-PLAN.md (task 2) + +--- + +### Inner Function Extraction for Clean Retry Wrapping + +When adding retry to existing code, extract the retryable operation into a small inner function, then wrap it with retry_with_meta() — minimal structural change, maximum clarity. + +**Source:** 16-02-PLAN.md (task 2), 16-02-SUMMARY.md + +--- + +### tqdm on stderr Preserves Pipe-Compatible Output + +progress_bar() writes to sys.stderr by default, ensuring stdout remains clean for piped data. The --no-progress flag provides explicit suppression for CI/non-TTY environments. + +**Source:** 16-02-PLAN.md (task 2), 16-02-SUMMARY.md + +--- + +## Surprises + +None documented. Both plans executed exactly as written with zero deviations. + +**Source:** 16-01-SUMMARY.md, 16-02-SUMMARY.md diff --git a/.planning/phases/17-dead-code-precommit/17-LEARNINGS.md b/.planning/phases/17-dead-code-precommit/17-LEARNINGS.md new file mode 100644 index 00000000..03149efd --- /dev/null +++ b/.planning/phases/17-dead-code-precommit/17-LEARNINGS.md @@ -0,0 +1,194 @@ +--- +phase: 17 +phase_name: "Dead Code Precommit" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 9 + lessons: 4 + patterns: 4 + surprises: 3 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +## Decisions + +### [tool.ruff] added targeting Python 3.10+ with E/F/I/UP/B/SIM rules + +Standard rule set selected: pycodestyle (E), pyflakes (F), isort (I), pyupgrade (UP), flake8-bugbear (B), flake8-simplify (SIM). Line-length set to 120 to match project conventions. + +**Source:** 17-01-PLAN.md (task 1, lines 154-165), 17-01-SUMMARY.md + +--- + +### B904 (raise-within-try) globally ignored + +Existing code intentionally uses bare `raise` in try/except blocks for read_json/write_json error paths. This is a deliberate pattern, not a bug. + +**Source:** 17-01-PLAN.md (task 1, line 161), 17-01-SUMMARY.md (decisions) + +--- + +### Pre-commit hooks NOT auto-installed (DX-04 deferred to Phase 18) + +The .pre-commit-config.yaml is created and documented in CONTRIBUTING.md, but hook installation (pre-commit install) is left as a manual step for Phase 18. + +**Source:** 17-01-PLAN.md (task 1, line 197), 17-01-SUMMARY.md (decisions) + +--- + +### Check 5 uses ast module for robust duplicate utility detection + +Rather than regex-based detection, Check 5 uses ast.parse to walk FunctionDef nodes and compare function names against the known _utils.py export list. + +**Source:** 17-01-PLAN.md (task 1, lines 203-215), 17-01-SUMMARY.md + +--- + +### per-file-ignores for pre-existing code quality issues not in scope + +106 pre-existing violations across tests/, scripts/, skills/, and some paperforge modules were suppressed via targeted per-file-ignores rather than fixing them in this phase. + +**Source:** 17-01-SUMMARY.md (Deviations from Plan, item 3) + +--- + +### load_vault_config delegation wrappers replaced with direct top-level imports + +All 7 worker modules: remove the 8-line delegation wrapper function, add `from paperforge.config import load_vault_config, paperforge_paths` at module level, and update intra-function import references. + +**Source:** 17-01-PLAN.md (task 2, lines 316-353), 17-01-SUMMARY.md + +--- + +### pipeline_paths function PRESERVED in each worker module + +Only the load_vault_config delegation wrapper was removed. The pipeline_paths function stays in each module because it extends the base paths with worker-only keys (17 extra keys). + +**Source:** 17-01-PLAN.md (task 2, lines 337-338), 17-01-SUMMARY.md + +--- + +### library_record field added to OCR error meta.json + +Both poll and upload error handlers now include `meta['library_record'] = key` to provide zotero_key context in error messages. Added to 4 error branches total. + +**Source:** 17-01-PLAN.md (task 1, lines 217-232), 17-01-SUMMARY.md + +--- + +### Dead system_dir/resources_dir/control_dir extraction removed from pipeline_paths + +ruff F841 flagged these expression-only statements (unused variables from old monolithic script) as dead code. Removed from all 7 worker modules along with the now-unused `cfg = load_vault_config(vault)` call inside pipeline_paths. + +**Source:** 17-01-SUMMARY.md (Deviations from Plan, item 1) + +--- + +## Lessons + +### ruff auto-fix can uncover dead code beyond just unused imports + +Beyond removing ~350 unused imports, ruff's F841 rule caught expression-only statements (dead system_dir/resources_dir/control_dir variable extractions) that were invisible to manual review. + +**Context:** These were remnants of the old monolithic script pattern that persisted through multiple refactoring phases. Only automated analysis flagged them. + +**Source:** 17-01-SUMMARY.md (Deviations - Rule 2, item 1) + +--- + +### Removing re-export wrappers cascades to dependent modules + +When _resolve_formal_note_path was removed from deep_reading.py's re-exports, repair.py broke because it imported the symbol through deep_reading.py rather than directly. The fix required changing repair.py's import to point directly to _utils.py. + +**Context:** This shows that even backward-compatible re-exports create implicit dependency chains that break when the re-export is removed. + +**Source:** 17-01-SUMMARY.md (Deviations - Rule 2, item 2) + +--- + +### per-file-ignores are necessary when doing targeted cleanup + +106 pre-existing violations across tests/, scripts/, skills/, and some paperforge modules were not in scope for this phase. Targeted per-file-ignores avoided scope creep while keeping ruff check clean. + +**Context:** These violations ranged from line-length issues to simplification suggestions in pre-existing test code that would have required disproportionate effort to fix. + +**Source:** 17-01-SUMMARY.md (Deviations - Rule 2, item 3) + +--- + +### Dead code accumulates silently across refactoring phases + +~450 total fixes (353 imports + 58 unsafe + 38 format + additional manual removals) were needed. Code that was "working" contained significant dead weight that had accumulated across multiple phases without detection. + +**Context:** The 7 worker modules, having been refactored multiple times (Phases 12, 14, 15, 16), accumulated unused imports from removed feature code, renamed APIs, and refactored utility functions. + +**Source:** 17-01-SUMMARY.md + +--- + +## Patterns + +### Pre-Commit Hooks as Safety Net + +Automated enforcement (ruff lint, ruff format, consistency audit) prevents code quality regressions at commit time rather than relying on manual review. + +**Source:** 17-01-PLAN.md (task 1), 17-01-SUMMARY.md + +--- + +### AST-Based Code Analysis for Structural Checks + +Using ast.parse rather than regex for Check 5's duplicate utility detection provides accurate function boundary detection without false positives from comments or strings. + +**Source:** 17-01-PLAN.md (task 1, lines 203-215), 17-01-SUMMARY.md + +--- + +### Direct Import Replacing Delegation Wrapper + +Remove the function hop by replacing `def load_vault_config(vault): from paperforge.config import load_vault_config as _shared; return _shared(vault)` with `from paperforge.config import load_vault_config` at module level. Same name binding, zero indirection. + +**Source:** 17-01-PLAN.md (task 2), 17-01-SUMMARY.md + +--- + +### Error Context Enrichment + +Adding a single field (`library_record` with zotero_key) to existing error data structures provides actionable debugging context without restructuring error handling paths. + +**Source:** 17-01-PLAN.md (task 1), 17-01-SUMMARY.md + +--- + +## Surprises + +### 353 issues auto-fixed by ruff -- far more unused imports than anticipated + +The initial ruff check found 353 issues automatable by --fix, plus 58 more via --unsafe-fixes. This vastly exceeded the expected number of dead imports. + +**Impact:** Required significantly more verification effort (module import testing, test suite) than planned. + +**Source:** 17-01-SUMMARY.md + +--- + +### Dead system_dir/resources_dir/control_dir extraction found by ruff via F841 + +Expression-only statements dating from the old monolithic script pattern were silently present in all 7 worker modules. Only ruff's F841 (unused-variable) rule flagged them. + +**Impact:** Required manual removal from all 7 worker modules plus adjusting pipeline_paths to no longer call load_vault_config (which was only needed for those dead extractions). + +**Source:** 17-01-SUMMARY.md (Deviations - Rule 2, item 1) + +--- + +### repair.py imported _resolve_formal_note_path through deep_reading.py's re-export, not directly + +When deep_reading.py's re-export of _resolve_formal_note_path was removed, repair.py broke. The implicit dependency via deep_reading.py was not documented and was invisible until import test failed. + +**Impact:** Required fixing repair.py to import directly from _utils.py. Highlighted the fragility of chained re-exports. + +**Source:** 17-01-SUMMARY.md (Deviations - Rule 2, item 2) diff --git a/.planning/phases/18-documentation-ux-polish/18-LEARNINGS.md b/.planning/phases/18-documentation-ux-polish/18-LEARNINGS.md new file mode 100644 index 00000000..9a2b0b82 --- /dev/null +++ b/.planning/phases/18-documentation-ux-polish/18-LEARNINGS.md @@ -0,0 +1,180 @@ +--- +phase: 18 +phase_name: "Documentation UX Polish" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 10 + lessons: 4 + patterns: 5 + surprises: 0 +missing_artifacts: + - "UAT.md" + - "VERIFICATION.md" +--- + +## Decisions + +### auto_analyze_after_ocr defaults to false (opt-in) + +Users must explicitly set `auto_analyze_after_ocr: true` in paperforge.json. This avoids unexpected behavioral changes — existing workflows remain unchanged until the user opts in. + +**Source:** 18-01-PLAN.md (task 1, line 77), 18-01-SUMMARY.md + +--- + +### Config option placed as top-level key in paperforge.json + +The new field sits alongside changelog_url at the top level, not nested under a sub-key. Simple access pattern consistent with other single-value settings. + +**Source:** 18-01-PLAN.md (task 1, line 77), 18-01-SUMMARY.md + +--- + +### Hook wrapped in try/except so a single failure does not abort OCR batch + +If the auto_analyze hook fails (e.g., library-record not found, JSON parse error), the error is logged as a warning and the OCR batch continues processing remaining items. + +**Source:** 18-01-PLAN.md (task 1, lines 84-101), 18-01-SUMMARY.md + +--- + +### CHANGELOG.md uses Keep a Changelog format + +Standardized format with [Unreleased], version headers, and Added/Changed/Fixed subsections. Ensures consistency and readability. + +**Source:** 18-01-PLAN.md (task 2), 18-01-SUMMARY.md + +--- + +### CHANGELOG.md does NOT update paperforge.json changelog_url + +The existing changelog_url in paperforge.json points to GitHub Releases for update checking. The new CHANGELOG.md is a local document for user reference — separate purposes, separate URLs. + +**Source:** 18-01-PLAN.md (task 2, line 169), 18-01-SUMMARY.md + +--- + +### ADR-012 documents _utils.py leaf module constraint from Phase 14 + +Formal architecture decision record capturing the leaf module constraint — _utils.py must never import from paperforge.worker.* or paperforge.commands.* — only from stdlib and paperforge.config. + +**Source:** 18-02-PLAN.md (task 2, lines 140-146), 18-02-SUMMARY.md + +--- + +### ADR-013 documents dual-output logging strategy from Phase 13 + +Formal architecture decision record capturing the stdout (user-facing) vs stderr (diagnostic) split, configure_logging() function, and PAPERFORGE_LOG_LEVEL env var pattern. + +**Source:** 18-02-PLAN.md (task 2, lines 148-154), 18-02-SUMMARY.md + +--- + +### AGENTS.md command mapping table follows v1.2 namespace conventions + +The "操作速查" table maps paperforge CLI commands to /pf-* agent commands, following the established v1.2+ naming convention. + +**Source:** 18-02-PLAN.md (task 2, lines 163-174), 18-02-SUMMARY.md + +--- + +### chart-reading INDEX ordered by biomedical commonness per D-04 + +19 guides ordered from most common (bar/column charts, forest plots) to least common (QQ plots). Guides not in D-04's primary list appended in logical groups. + +**Source:** 18-02-PLAN.md (task 3, lines 201-243), 18-02-SUMMARY.md + +--- + +### README.md orphaned code lines removed rather than enclosed in fences + +Three orphaned lines (legacy python commands) and their lone closing backtick were removed outright, letting the quote block flow directly to the License section. + +**Source:** 18-02-PLAN.md (task 1, lines 113-118), 18-02-SUMMARY.md + +--- + +## Lessons + +### OCR hook placement matters: must run BEFORE selection-sync + +The auto_analyze hook inserts at line 1499 (after ocr_status=done) but before _sync.run_selection_sync(vault) at line ~1589. This ensures the frontmatter change to analyze:true is not overwritten by the sync. + +**Context:** selection-sync does not overwrite user-controlled fields like analyze, but placing the hook after sync would still cause a timing issue where the user would see the record before the auto-update. + +**Source:** 18-01-PLAN.md (task 1, lines 108-109), 18-01-SUMMARY.md + +--- + +### Existing re and read_json imports eliminated need for new imports in ocr.py + +Both `re` and `read_json` (from _utils.py) were already imported in ocr.py. The auto_analyze hook required no new import statements, simplifying the change. + +**Source:** 18-01-PLAN.md (task 1, line 105), 18-01-SUMMARY.md + +--- + +### Keep a Changelog format provides clear structure with minimal maintenance + +The format's Added/Changed/Fixed subsections and version-header structure makes it easy to review release history without over-documenting individual commits. + +**Source:** 18-01-PLAN.md (task 2), 18-01-SUMMARY.md + +--- + +### Using v1.2 migration doc as template for v1.4 migration doc ensured consistency + +Following the established structure (What's New, Breaking Changes, Detailed sections, Migration Steps, Rollback, FAQ) from MIGRATION-v1.2.md provided a familiar format for users upgrading from v1.3. + +**Source:** 18-02-PLAN.md (task 1, line 97), 18-02-SUMMARY.md + +--- + +## Patterns + +### try/except Guard for Non-Critical Automation Hooks + +The auto_analyze hook wraps its entire body in try/except so failure (missing file, parse error, etc.) is logged but never aborts the parent operation (OCR batch). + +**Source:** 18-01-PLAN.md (task 1), 18-01-SUMMARY.md + +--- + +### re.sub with MULTILINE Flag for Frontmatter Field Substitution + +Using `re.sub(r"^analyze:.*$", "analyze: true", text, count=1, flags=re.MULTILINE)` for frontmatter field replacement — same pattern used by deep_reading.py's status sync. + +**Source:** 18-01-PLAN.md (task 1, lines 93-98), 18-01-SUMMARY.md + +--- + +### ADR Format Consistently Applied Across Architecture Decisions + +All 13 ADRs follow the same format (Status, Phase, Context, Decision, Consequences), providing consistent reference regardless of when the decision was made. + +**Source:** 18-02-PLAN.md (task 2), 18-02-SUMMARY.md + +--- + +### Migration Doc Template Reuse + +MIGRATION-v1.4.md uses the same structure as MIGRATION-v1.2.md (What's New, Breaking Changes, Detailed sections, Migration Steps, Rollback, FAQ). This provides a familiar experience for users who previously upgraded. + +**Source:** 18-02-PLAN.md (task 1), 18-02-SUMMARY.md + +--- + +### Command Mapping Table as User Reference + +The "操作速查" table in AGENTS.md maps user intent ("what you want to do") to both the terminal command and the Agent command, serving as a quick reference for users switching between interfaces. + +**Source:** 18-02-PLAN.md (task 2), 18-02-SUMMARY.md + +--- + +## Surprises + +None documented. Both plans executed exactly as written with zero deviations. + +**Source:** 18-01-SUMMARY.md, 18-02-SUMMARY.md diff --git a/.planning/phases/19-testing/19-LEARNINGS.md b/.planning/phases/19-testing/19-LEARNINGS.md new file mode 100644 index 00000000..8aba7e08 --- /dev/null +++ b/.planning/phases/19-testing/19-LEARNINGS.md @@ -0,0 +1,154 @@ +--- +phase: 19 +phase_name: "Testing" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 7 + lessons: 3 + patterns: 3 + surprises: 2 +missing_artifacts: + - "UAT.md" +--- + +## Decisions + +### Test file per function group +Created 4 dedicated unit test files covering all `paperforge/worker/_utils.py` functions, plus `test_e2e_pipeline.py` and `test_setup_wizard.py`. Each test file covers one logical module/function group. + +**Rationale/Context:** Close TEST-04 (last testing gap in v1.4 code health phase) and TEST-01/TEST-02 (E2E and setup wizard coverage). Isolated test files make failures easier to diagnose and maintain. + +**Source:** 19-01-PLAN.md, 19-02-PLAN.md, 19-03-PLAN.md + +--- + +### Cache reset pattern for journal DB tests +Journal DB tests must reset `_utils._JOURNAL_DB = None` between test scenarios to account for the module-level global cache. + +**Rationale/Context:** `_utils.py` caches `_JOURNAL_DB` globally across calls within the same process. Without explicit cache reset, tests that check loading behavior would get stale cached results. + +**Source:** 19-01-PLAN.md Task 4 + +--- + +### E2E tests run against existing fixture vault +Rather than deleting and recreating records, E2E tests run selection-sync on the existing `test_vault` fixture and verify the updated records match expected values. + +**Rationale/Context:** The `test_vault` fixture already creates library records and formal notes. Running sync on the existing vault verifies deduplication behavior and updates, which is more realistic than testing from scratch. + +**Source:** 19-02-PLAN.md Task 1 + +--- + +### State-based OCR validation +OCR pipeline behavior is validated via meta.json state files only, not by calling `run_ocr()`. + +**Rationale/Context:** `run_ocr()` requires network access to PaddleOCR API. State-based validation (`ocr_status: done/pending/nopdf`) avoids external dependencies and is faster. + +**Source:** 19-02-PLAN.md Task 1, 19-02-SUMMARY.md + +--- + +### sys.path insertion for setup_wizard tests +Test files for `setup_wizard.py` insert repo root into `sys.path` to make the module importable. + +**Rationale/Context:** Same pattern as `test_smoke.py`. The repo root is not in the default Python path, so explicit insertion is needed for imports to work. + +**Source:** 19-03-PLAN.md Task 1 + +--- + +### No Textual UI testing +Tests focus on standalone functions and classes (`AGENT_CONFIGS`, `EnvChecker`, `CheckResult`, `_find_vault`) — the `SetupWizardApp` Textual App class is excluded. + +**Rationale/Context:** Textual-based App and StepScreen classes require a running terminal. Focusing on standalone detection/validation logic allows testing without terminal dependencies. + +**Source:** 19-03-PLAN.md + +--- + +### 3-stage pipeline consistency check +E2E tests validate consistency across selection-sync, index-refresh, formal notes, and OCR states in a single test method. + +**Rationale/Context:** A `test_full_pipeline_consistency()` test runs all stages sequentially and cross-validates outputs (matching titles, consistent ocr_status, correct wikilinks), ensuring the pipeline produces coherent state end-to-end. + +**Source:** 19-02-PLAN.md, 19-02-SUMMARY.md + +--- + +## Lessons + +### yaml_list() None filter bug discovered during test writing +`yaml_list()` filter in `_utils.py:110` — `str(None)` yielded `"None"` (truthy) instead of being filtered out. + +**Context:** While writing unit tests for `yaml_list`, it was discovered that `None` values in list items were not properly filtered because `str(None)` produces the string `"None"` which is truthy. Fixed with explicit `value is not None` guard. + +**Source:** 19-01-SUMMARY.md + +--- + +### High test count growth from utility tests +Starting from ~205 existing tests, adding 71 new tests across 4 utility test files resulted in 317 total tests. + +**Context:** The 4 unit test files added 71 tests (21+22+20+8), plus E2E and setup wizard tests adding 13+30 = 43 more, for a total of 114 new tests reaching 317 passed, 2 skipped. + +**Source:** 19-01-SUMMARY.md, 19-02-SUMMARY.md, 19-03-SUMMARY.md + +--- + +### Zero regression consistently achieved +All 3 sub-phases added 0 regressions to the existing test suite. + +**Context:** Each of the 3 test sub-plans explicitly verified zero regressions. The full suite passed with 317 tests (2 skipped) at the end of the phase. + +**Source:** 19-01-SUMMARY.md, 19-02-SUMMARY.md, 19-03-SUMMARY.md + +--- + +## Patterns + +### tmp_path for isolated vault creation +All tests use pytest's `tmp_path` fixture for temporary directory creation instead of heavyweight vault fixtures. + +**When to use:** For unit tests that need temporary files or directories but don't need a full vault structure with paperforge.json and exports. + +**Source:** 19-01-PLAN.md, 19-02-PLAN.md, 19-03-PLAN.md + +--- + +### conftest fixtures for E2E tests +`test_vault` fixture provides a full vault with paperforge.json, exports, OCR fixtures, and library records. + +**When to use:** For integration tests that need the complete vault structure including paperforge.json configuration, Zotero exports, OCR state files, and library records. + +**Source:** 19-02-PLAN.md + +--- + +### monkeypatch/patch for platform-dependent tests +`unittest.mock.patch` and monkeypatch used for import simulation, path override, and platform-dependent behavior. + +**When to use:** When testing code that depends on environment state (Zotero installation, Python version, current working directory). + +**Source:** 19-03-PLAN.md, 19-03-SUMMARY.md + +--- + +## Surprises + +### Fast execution speed +71 new tests created across 3 sub-plans in ~16 minutes total (8+4+4 min). + +**Impact:** The testing phase was completed much faster than expected, demonstrating that well-specified test plans with detailed code snippets speed implementation significantly. + +**Source:** 19-01-SUMMARY.md, 19-02-SUMMARY.md, 19-03-SUMMARY.md + +--- + +### Bug found while writing tests, not in production +The yaml_list None filter bug was discovered during test creation rather than in production usage. + +**Impact:** The test-driven approach caught a latent bug that would have caused incorrect YAML output when None values appeared in list data. Proves the value of comprehensive unit test coverage. + +**Source:** 19-01-SUMMARY.md diff --git a/.planning/phases/20-plugin-settings-shell-persistence/20-LEARNINGS.md b/.planning/phases/20-plugin-settings-shell-persistence/20-LEARNINGS.md new file mode 100644 index 00000000..7cdcf7ae --- /dev/null +++ b/.planning/phases/20-plugin-settings-shell-persistence/20-LEARNINGS.md @@ -0,0 +1,154 @@ +--- +phase: 20 +phase_name: "Plugin Settings Shell & Persistence" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 6 + lessons: 3 + patterns: 3 + surprises: 3 +missing_artifacts: + - "UAT.md" +--- + +## Decisions + +### All code in main.js +All settings code stays in `main.js` with no build system — Obsidian's `require('obsidian')` works natively, and a second file would add path complexity. + +**Rationale/Context:** CommonJS `require` from Obsidian already works in the plugin context. A separate settings file would need careful path handling relative to the plugin directory. Keeping everything in one file is simpler and avoids build tooling. + +**Source:** 20-PLAN.md, 20-SUMMARY.md + +--- + +### Debounced save at 500ms +In-memory settings update immediately on input change; disk write is debounced via `setTimeout`/`clearTimeout` with 500ms delay. + +**Rationale/Context:** Prevents thrashing `data.json` on every keystroke while ensuring responsive UI. User sees instant feedback; disk writes are batched. + +**Source:** 20-PLAN.md, 20-SUMMARY.md + +--- + +### String fields only +All 8 settings are text inputs; no toggles/selects needed. Password field type for API key. + +**Rationale/Context:** All configuration values in this phase (vault_path, system_dir, resources_dir, etc.) are path strings or API keys. No boolean toggles or dropdown selections are required. + +**Source:** 20-PLAN.md + +--- + +### display() lifecycle for tab switch survival +`display()` reconstructs DOM from `this.plugin.settings` on each call (tab switch). In-memory settings preserve state with zero data loss. + +**Rationale/Context:** Obsidian's settings tab calls `display()` each time the tab is shown. By rebuilding from the in-memory settings object (not re-reading disk), typed values survive tab switches without disk I/O on every transition. + +**Source:** 20-PLAN.md, 20-SUMMARY.md + +--- + +### Null-safe merge for fresh install +`Object.assign({}, DEFAULTS, await this.loadData())` prevents TypeError on fresh install when `loadData()` returns null. + +**Rationale/Context:** Obsidian's `loadData()` returns `null` when no prior `data.json` exists. `Object.assign` silently ignores null/undefined sources, so the merge produces defaults without a TypeError guard check. + +**Source:** 20-PLAN.md, 20-SUMMARY.md, 20-VERIFICATION.md + +--- + +### No CSS additions needed +Obsidian's built-in `.setting-item` styles render the settings tab perfectly; no custom styles needed. + +**Rationale/Context:** Obsidian's settings UI framework provides complete styling for setting items (labels, descriptions, input fields, sections). Custom CSS would be redundant. + +**Source:** 20-SUMMARY.md, 20-VERIFICATION.md + +--- + +## Lessons + +### Tasks 1+2 are functionally interdependent +Both modify `main.js` and share state (tab UI calls `plugin.settings` and `plugin.saveSettings()`), so they were combined into one atomic commit. + +**Context:** Task 1 (data model + persistence methods) and Task 2 (settings tab UI) both modify the same class and cannot be tested independently. The tab UI immediately references methods added in Task 1. + +**Source:** 20-SUMMARY.md + +--- + +### Obsidian PluginSettingTab API is straightforward +Extending `PluginSettingTab` with the `Setting` form builder works well and requires minimal boilerplate. + +**Context:** The entire settings tab with 8 fields in 3 sections was implemented in ~87 lines using Obsidian's built-in settings API. No custom HTML or event handling needed. + +**Source:** 20-SUMMARY.md + +--- + +### Requirements tracking metadata can drift from implementation +The SUMMARY (`requirements-completed: [SETUP-01, SETUP-02]`) and REQUIREMENTS.md both show SETUP-03 as unchecked, but the actual code in `main.js` fully implements SETUP-03. + +**Context:** The code was written and committed implementing immediate in-memory updates, debounced disk writes, and tab switch survival — all satisfying SETUP-03. But the tracking metadata was not updated to reflect this. + +**Source:** 20-VERIFICATION.md + +--- + +## Patterns + +### Debounced persistence pattern +`clearTimeout`/`setTimeout` wrapper with 500ms delay for debounced disk writes on field change. + +**When to use:** When you need instant UI responsiveness but want to batch/rate-limit expensive persistence operations. Applicable to any Obsidian plugin with editable settings. + +**Source:** 20-SUMMARY.md, 20-VERIFICATION.md + +--- + +### Null-safe merge pattern +`Object.assign({}, DEFAULTS, await this.loadData())` as a safe initialization pattern for Obsidian plugin settings. + +**When to use:** When merging Obsidian plugin defaults with persisted settings. Handles fresh install (null/undefined), partial settings (missing keys get defaults), and full settings (all values override). + +**Source:** 20-SUMMARY.md, 20-VERIFICATION.md + +--- + +### In-memory state on change + deferred disk write +Immediate memory update for responsiveness, 500ms debounce for persistence. + +**When to use:** When you need low-latency UI feedback but want to avoid excessive disk I/O. The in-memory object serves as the single source of truth during the session; disk is only a persistence layer. + +**Source:** 20-SUMMARY.md + +--- + +## Surprises + +### Very fast execution +2 minutes to complete the entire phase (3 tasks, 87 lines added to main.js). + +**Impact:** The phase was substantially faster than anticipated because the plan was well-specified and the Obsidian API had no surprises. The code already existed in the working tree matching the plan. + +**Source:** 20-SUMMARY.md + +--- + +### No CSS changes needed +Obsidian's default setting-item styles handle the settings tab perfectly without any custom CSS. + +**Impact:** Eliminated an entire task from the plan. The Obsidian settings UI framework is more comprehensive than initially assumed. + +**Source:** 20-SUMMARY.md + +--- + +### Metadata gap between tracking and implementation +SUMMARY and REQUIREMENTS.md disagree on SETUP-03 status (tracked as unchecked but actually implemented in code). + +**Impact:** This could lead to incorrect status reports and confusion about what's actually implemented. Highlights the need for automated verification that cross-references tracking metadata against actual code. + +**Source:** 20-VERIFICATION.md diff --git a/.planning/phases/21-one-click-install-and-polished-ux/21-LEARNINGS.md b/.planning/phases/21-one-click-install-and-polished-ux/21-LEARNINGS.md new file mode 100644 index 00000000..0abfa330 --- /dev/null +++ b/.planning/phases/21-one-click-install-and-polished-ux/21-LEARNINGS.md @@ -0,0 +1,226 @@ +--- +phase: 21 +phase_name: "One-Click Install and Polished UX" +project: "PaperForge Lite" +generated: "2026-05-02" +counts: + decisions: 10 + lessons: 4 + patterns: 5 + surprises: 4 +missing_artifacts: + - "UAT.md" +--- + +## Decisions + +### Additive-only approach to settings tab +All additions to `PaperForgeSettingTab.display()` are purely additive — no modifications to `PaperForgeStatusView` sidebar or `ACTIONS[]` definitions. + +**Rationale/Context:** Zero-regression guarantee. The install button exists in the settings tab only and must not affect existing sidebar functionality or command palette actions (INST-04 requirement). + +**Source:** 21-01-PLAN.md, 21-01-SUMMARY.md, 21-VERIFICATION.md + +--- + +### Client-side field validation only +Uses simple string trim checks (`!s.key || !s.key.trim()`), no filesystem calls. + +**Rationale/Context:** Initial consideration included `fs.existsSync()` for path validation, but that requires Node sync I/O which blocks the event loop. Non-empty validation is sufficient to prevent empty-field subprocess spawns. Path validation is deferred to the subprocess which fails gracefully with a friendly error message. + +**Source:** 21-01-PLAN.md, 21-01-SUMMARY.md + +--- + +### zotero_data_dir excluded from validation +The Zotero data directory field is optional and NOT checked by `_validate()`. + +**Rationale/Context:** Zotero data dir is auto-detected by `headless_setup()` when not provided. Making it optional reduces friction for users who don't need to override it. + +**Source:** 21-01-PLAN.md, 21-01-SUMMARY.md + +--- + +### All error messages in Chinese +Error messages from `_validate()`, `_formatSetupError()`, and `_showNotice()` are all in Chinese. + +**Rationale/Context:** Required by INST-03. The target user base is Chinese-speaking medical researchers. Even technical error messages (Python not found, module missing) are localized. + +**Source:** 21-01-SUMMARY.md, 21-02-PLAN.md, 21-02-SUMMARY.md + +--- + +### spawn over exec for subprocess +Use `node:child_process.spawn` (not `exec`) for non-blocking subprocess execution with stdout streaming. + +**Rationale/Context:** `spawn` streams stdout line-by-line so `_processSetupOutput()` can show step progress in real-time. `exec` buffers all output until completion — no intermediate feedback possible. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +### --headless flag, not --non-interactive +The correct CLI flag for headless setup is `--headless`, not `--non-interactive`. + +**Rationale/Context:** The CLI interface defines `--headless` as the flag for non-interactive mode. The initial plan draft incorrectly used `--non-interactive` — corrected during code review. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +### API key via --paddleocr-key CLI flag +API key is passed via `--paddleocr-key` CLI argument, not from `PADDLEOCR_API_TOKEN` env var. + +**Rationale/Context:** The `headless_setup()` function reads API key from the CLI argument (`cli.py` line 424-440), not from environment variable. Confirmed by reading the actual Python CLI code. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md + +--- + +### Explicit directory args override defaults +Plugin defaults (20_Resources, Control) must override headless_setup's built-in defaults (03_Resources, LiteratureControl) via --resources-dir and --control-dir. + +**Rationale/Context:** The plugin's DEFAULT_SETTINGS use different defaults for resources_dir and control_dir than the Python headless_setup function. Without explicit override, the subprocess would create directories in different locations than what the plugin expects. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md + +--- + +### try/finally for button lifecycle +Button is disabled BEFORE `await`, re-enabled in `finally` — guarantees no double-click even if spawn throws synchronously. + +**Rationale/Context:** Prevents race conditions. If `spawn()` throws synchronously, the `finally` block still runs and re-enables the button. The button text also changes to "正在安装..." during execution for visual feedback. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +### 5 error patterns mapped to Chinese +`_formatSetupError()` matches 5 common subprocess error patterns: Python not found, module missing, permission denied, path not found, timeout. + +**Rationale/Context:** Covers the most common failure modes for first-time users (no Python, no pip install, file permissions, wrong path, network timeout). Full error always available via `console.error()` for debugging. Fallback truncates stderr to 3 lines, 200 chars. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +## Lessons + +### Obsidian CSS variables provide automatic theme compatibility +Using `--radius-m`, `--font-ui-small`, `--background-secondary`, `--color-green`, `--text-error`, `--color-blue` ensures install status styles work in both light and dark mode. + +**Context:** By using Obsidian's built-in CSS variables rather than hardcoded colors, the install status area automatically adapts to the user's theme without any additional CSS work. + +**Source:** 21-01-PLAN.md Task 3 + +--- + +### color-mix() creates clean tinted backgrounds +`color-mix(in srgb, var(--color-green) 10%, var(--background-secondary))` creates subtle tinted backgrounds without overlapping full theme colors. + +**Context:** The 8-10% opacity creates a gentle tint that indicates status (green/red/blue) without being visually overwhelming. This is cleaner than using `opacity` on the entire element. + +**Source:** 21-01-PLAN.md, 21-01-SUMMARY.md, 21-VERIFICATION.md + +--- + +### headless_setup has different defaults than plugin +The Python `headless_setup()` function's built-in defaults (resources_dir="03_Resources", control_dir="LiteratureControl") differ from the plugin's DEFAULT_SETTINGS (resources_dir="20_Resources", control_dir="Control"). + +**Context:** Without explicit `--resources-dir` and `--control-dir` args in the spawn call, the subprocess would create directories in locations the plugin doesn't expect, causing silent misconfiguration. + +**Source:** 21-02-PLAN.md + +--- + +### Pre-existing code has raw stderr in Notice +Line 484 of `main.js` (pre-existing, not from Phase 21) exposes raw stderr in `new Notice`. + +**Context:** The verification report flagged a pre-existing anti-pattern in the command palette actions dispatch code. Phase 21's `_showNotice()` in PaperForgeSettingTab correctly uses `_formatSetupError()` to avoid raw error exposure. This pre-existing issue is outside Phase 21 scope. + +**Source:** 21-VERIFICATION.md + +--- + +## Patterns + +### Obsidian CSS variables for theme consistency +All install status styling uses Obsidian's built-in CSS variables for automatic light/dark theme compatibility. + +**When to use:** Any Obsidian plugin CSS should use Obsidian CSS variables (`--background-secondary`, `--color-green`, `--text-error`, etc.) instead of hardcoded colors. + +**Source:** 21-01-PLAN.md, 21-01-SUMMARY.md, 21-VERIFICATION.md + +--- + +### color-mix() for status backgrounds +Three color variants using `color-mix(in srgb, %, var(--background-secondary))` for tinted backgrounds. + +**When to use:** When you need subtly colored status indicators that blend with the existing theme. The 8-10% range provides visible tint without being distracting. + +**Source:** 21-01-PLAN.md, 21-VERIFICATION.md + +--- + +### spawn (not exec) for non-blocking subprocess +Use `spawn` when you need real-time progress updates from a long-running subprocess. + +**When to use:** Any time a plugin spawns a subprocess that produces stepped output and you want to show progress to the user. `spawn` streams stdout line-by-line; `exec` buffers everything. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +### try/finally for guaranteed resource cleanup +Button disable/enable wrapped in try/finally to ensure resources are always released. + +**When to use:** Any async operation that acquires a resource (disabling a button, showing a modal, locking a state). The finally block guarantees cleanup whether the operation succeeds, fails, or throws synchronously. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +### Pattern-based error mapping +Regex patterns mapped to user-facing Chinese messages for common subprocess error codes. + +**When to use:** When a subprocess can fail with cryptic system error messages (ENOENT, EACCES, ModuleNotFoundError) that need to be translated into user-friendly text. Pattern matching captures the error class while the full message is logged to console for debugging. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md, 21-VERIFICATION.md + +--- + +## Surprises + +### Headless setup CLI flag confusion +Initially planned to use `--non-interactive`, but the correct CLI flag was `--headless`. + +**Impact:** Could have caused silent failure (CLI would ignore unrecognized flags). Highlights the importance of reading the actual Python CLI code (cli.py) rather than guessing flag names from context. + +**Source:** 21-02-PLAN.md, 21-02-SUMMARY.md + +--- + +### Non-obvious default differences between plugin and CLI +headless_setup's built-in defaults (03_Resources, LiteratureControl) differ from plugin defaults (20_Resources, Control). + +**Impact:** This would cause the subprocess to create directories in unexpected locations, appearing to succeed but actually misconfiguring the vault. Required explicit --resources-dir and --control-dir overrides. + +**Source:** 21-02-PLAN.md + +--- + +### Pre-existing raw error exposure in command palette actions +Existing code line 484 exposed raw stderr in `new Notice`, which Phase 21's careful error handling specifically avoids. + +**Impact:** Phase 21's INST-02 compliance (no raw tracebacks in user-facing messages) is stricter than the existing codebase. Highlights the need to audit pre-existing error handling patterns. + +**Source:** 21-VERIFICATION.md + +--- + +### Validation approach simplified from initial plan +Initial plan considered `fs.existsSync()` for path existence checks but simplified to just non-empty validation since subprocess handles path validation. + +**Impact:** Saved complexity (no sync I/O, no error handling for filesystem calls) while still preventing the main failure case (empty fields). The subprocess path validation is then mapped to a friendly Chinese message. + +**Source:** 21-01-PLAN.md Task 2 diff --git a/.planning/phases/22-install-wizard-modal/22-PLAN.md b/.planning/phases/22-configuration-truth-compatibility/22-00-PLAN.md similarity index 100% rename from .planning/phases/22-install-wizard-modal/22-PLAN.md rename to .planning/phases/22-configuration-truth-compatibility/22-00-PLAN.md diff --git a/.planning/phases/22-configuration-truth-compatibility/22-00-SUMMARY.md b/.planning/phases/22-configuration-truth-compatibility/22-00-SUMMARY.md new file mode 100644 index 00000000..08d16667 --- /dev/null +++ b/.planning/phases/22-configuration-truth-compatibility/22-00-SUMMARY.md @@ -0,0 +1,9 @@ +--- +phase: 22 +title: Install Wizard Modal (merged with configuration truth) +plan: 22-00-PLAN.md +status: complete +actual_effort: 0 (already covered by 22-01/02/03) +--- + +Install wizard modal planning document was merged into phase 22. All implementation work was covered by 22-01 (plugin settings persistence), 22-02 (modal rendering), and 22-03 (install integration). diff --git a/.planning/phases/22-configuration-truth-compatibility/22-CONTEXT.md b/.planning/phases/22-configuration-truth-compatibility/22-CONTEXT.md index 9418edae..d6dd8576 100644 --- a/.planning/phases/22-configuration-truth-compatibility/22-CONTEXT.md +++ b/.planning/phases/22-configuration-truth-compatibility/22-CONTEXT.md @@ -1,7 +1,7 @@ # Phase 22: Configuration Truth & Compatibility - Context **Gathered:** 2026-05-03 -**Status:** Ready for planning +**Status:** Complete ## Phase Boundary diff --git a/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-PLAN.md b/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-PLAN.md new file mode 100644 index 00000000..58693f2a --- /dev/null +++ b/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-PLAN.md @@ -0,0 +1,298 @@ +--- +phase: 23-canonical-asset-index-safe-rebuilds +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - pyproject.toml + - paperforge/worker/asset_index.py + - paperforge/worker/sync.py + - paperforge/worker/_utils.py + - tests/test_asset_index.py +autonomous: true +requirements: + - ASSET-01 + - ASSET-02 + - ASSET-04 + - MIG-02 + +must_haves: + truths: + - "asset_index.py exists and exports build_index(vault), get_index_path(vault), atomic_write_index(path, data)" + - "build_index produces a versioned envelope: {schema_version, generated_at, paper_count, items}" + - "Index writes use tempfile + os.replace() so interrupted writes never leave half-written files" + - "Index writes use filelock cross-process lock (10s timeout) so concurrent sync/ocr/repair do not corrupt the index" + - "run_index_refresh() in sync.py delegates to asset_index.build_index()" + - "filelock is listed in pyproject.toml dependencies" + artifacts: + - path: "pyproject.toml" + provides: "filelock dependency" + contains: "filelock" + - path: "paperforge/worker/asset_index.py" + provides: "Core index module" + exports: "build_index, get_index_path, atomic_write_index" + - path: "paperforge/worker/sync.py" + provides: "Delegation point" + modifies: "run_index_refresh delegates to asset_index.build_index" + - path: "tests/test_asset_index.py" + provides: "Test coverage" + min_lines: 80 + key_links: + - from: "sync.py :: run_index_refresh()" + to: "asset_index.py :: build_index()" + via: "delegation call" + - from: "asset_index.py :: build_index()" + to: "_utils.py :: pipeline_paths()" + via: "paths['index'] for write destination" + - from: "asset_index.py :: atomic_write_index()" + to: "tempfile.NamedTemporaryFile + os.replace + filelock" + via: "atomic write implementation" + +--- + + +Create the `paperforge/worker/asset_index.py` module as the single canonical home for literature asset index generation. Implement versioned envelope format, atomic writes with cross-process locking (filelock), and full-rebuild logic extracted from sync.py. + +Purpose: Extract index generation from the bloated sync.py, add graceful atomic-write safety (Windows-compatible), and lock-based concurrency protection so that sync/ocr/repair won't corrupt the index when they write concurrently. + +Output: `asset_index.py` with `build_index()`, `get_index_path()`, `atomic_write_index()` + `sync.py` delegation + `filelock` dependency + tests. + + + +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md + + + +@.planning/STATE.md +@.planning/ROADMAP.md (Phase 23) +@.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-01..D-07, D-15..D-17, D-20) +@paperforge/worker/sync.py (lines 1669-1792 -- current run_index_refresh) +@paperforge/worker/_utils.py (lines 241-265 -- pipeline_paths including "index") +@paperforge/config.py (lines 49-58 -- DEFAULT_CONFIG, schema_version) +@pyproject.toml (lines 27-34 -- current dependencies) + + + + + + Task 1: Add filelock dependency and install + pyproject.toml + + + @pyproject.toml (lines 27-34 -- dependencies list) + + + + ADD `"filelock>=3.13.0"` to the dependencies list in pyproject.toml, after the existing tqdm line. + + THEN run: + ```bash + pip install -e . + ``` + to install the new dependency. Verify with: + ```bash + python -c "import filelock; print(filelock.__version__)" + ``` + + + + python -c "import filelock; print(filelock.__version__)" + + + + - pyproject.toml contains `"filelock>=3.13.0"` in the dependencies section + - `python -c "import filelock"` exits 0 + + + filelock 3.13+ installed and importable + + + + Task 2: Create paperforge/worker/asset_index.py with envelope, atomic writes, and build_index extraction + + paperforge/worker/asset_index.py + paperforge/worker/sync.py + + + + @paperforge/worker/sync.py (lines 1669-1746 -- current run_index_refresh: exports loop, entry construction, write_json call) + @paperforge/worker/sync.py (lines 1-30 -- imports at top of file) + @paperforge/worker/_utils.py (lines 241-265 -- pipeline_paths including "index" at line 263) + @paperforge/worker/_utils.py (lines 69-71 -- current write_json for comparison) + @paperforge/config.py (lines 105-112 -- get_paperforge_schema_version) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-01..D-07, D-15..D-17, D-20) + + + + Create `paperforge/worker/asset_index.py` as a new module with the following design: + + **Constants at module top:** + ```python + CURRENT_SCHEMA_VERSION = "2" + INDEX_FILENAME = "formal-library.json" + LOCK_TIMEOUT = 10 # seconds + ``` + + **Function: `get_index_path(vault: Path) -> Path`** + - Calls `paperforge_paths(vault)` from `paperforge.config` + - Returns `paths["paperforge"] / "indexes" / INDEX_FILENAME` + - This is the same path that `pipeline_paths()` currently returns via `paths["index"]` + + **Function: `atomic_write_index(path: Path, data: dict) -> None`** + - Uses `filelock.FileLock` with a `.lock` sibling file (e.g., path.with_suffix(".json.lock")) + - Lock timeout: 10 seconds; on timeout raises `filelock.Timeout` with a clear error message + - Writes to `tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False, dir=path.parent)` — uses `dir=path.parent` so rename is same-filesystem (Windows-safe) + - Serializes data with `json.dumps(data, ensure_ascii=False, indent=2)` + - Calls `os.replace(temp.name, path)` — atomic on both POSIX and Windows + - Handles cleanup: if the write fails before os.replace, remove the temp file + - Path parent directory is created if it does not exist (path.parent.mkdir(parents=True, exist_ok=True)) + + **Function: `build_envelope(items: list[dict]) -> dict`** + - Returns the versioned envelope (per D-02): + ```python + { + "schema_version": CURRENT_SCHEMA_VERSION, + "generated_at": datetime.utcnow().isoformat() + "Z", + "paper_count": len(items), + "items": items, + } + ``` + + **Function: `build_index(vault: Path) -> int`** + - Extracted from the loop body in `sync.py` `run_index_refresh()` (lines 1686-1744) + - Signature: `def build_index(vault: Path, verbose: bool = False) -> int` + - Imports internally: + - `from paperforge.worker._utils import pipeline_paths, read_json, slugify_filename` + - `from paperforge.worker.base_views import ensure_base_views` + - `from paperforge.worker.ocr import validate_ocr_meta` + - `from paperforge.worker.sync import load_domain_config, load_export_rows, collection_fields, obsidian_wikilink_for_pdf, obsidian_wikilink_for_path, has_deep_reading_content, frontmatter_note` + - **Logic (preserve existing behavior from sync.py lines 1673-1744):** + 1. Call `paths = pipeline_paths(vault)`, `config = load_domain_config(paths)`, `ensure_base_views(vault, paths, config)` + 2. Export path key-value: iterate export JSON files, group rows by domain + 3. Loop over export items, build the entry dict with the same fields as the current code (lines 1711-1740) + 4. For each entry, write/update the formal note via `frontmatter_note(entry, existing_text)` + 5. Append entry to a list + 6. After the loop, call `atomic_write_index(paths["index"], build_envelope(items))` + 7. Print `f"index-refresh: wrote {len(items)} index rows"` + 8. Return the item count + - Note: The orphaned-record cleanup (lines 1747-1791 of sync.py) stays in sync.py's `run_index_refresh()` — only the core build loop moves into `build_index()` + + **Modify `sync.py` `run_index_refresh()`:** + - Import `asset_index` at the top of sync.py + - Replace the loop body (lines 1686-1745) with a single call: + ```python + count = asset_index.build_index(vault, verbose) + # Existing orphaned-record cleanup follows (lines 1747-1791) + ``` + - Keep the existing orphaned-record cleanup logic in sync.py + - Keep the existing base_views / domain_config setup that's already in run_index_refresh (lines 1670-1675) + + **Important implementation details:** + - DO NOT delete the existing `run_index_refresh` function from sync.py — it still handles base views and orphan cleanup around the delegate call + - DO keep all imports that the delegated function needs locally within `build_index` (lazy imports to avoid circular deps) + - The `build_index` function is self-contained: it accepts `vault` and does everything needed to produce the index + - The current `_utils.py` `write_json()` function is NOT removed — other code still uses it for non-index files + - ALL paths (`paths["index"]`) come from the existing `pipeline_paths()` function, which is unchanged + + **Edge cases to handle:** + - Empty export directory: return 0, write empty envelope `{"schema_version": "2", "generated_at": "...", "paper_count": 0, "items": []}` + - Export JSON with no items: same as empty + - File lock contention: raise clear `filelock.Timeout` error message with instructions to retry + - Windows tempfile + os.replace: using same-directory tempfile via `dir=path.parent` ensures cross-filesystem rename issues don't occur + + + + python -c "from paperforge.worker.asset_index import build_index, get_index_path, atomic_write_index; print('OK')" + + + + - `paperforge/worker/asset_index.py` exists and is importable + - Module exports `build_index`, `get_index_path`, `atomic_write_index` + - `sync.py` `run_index_refresh()` still exists and calls `asset_index.build_index()` + - `python -c "from paperforge.worker.asset_index import build_index, get_index_path, atomic_write_index; print('OK')"` exits 0 + - `atomic_write_index` uses `filelock.FileLock` and `tempfile.NamedTemporaryFile` (grep for both in asset_index.py) + - `build_index` builds envelope with `schema_version`, `generated_at`, `paper_count`, `items` keys + + + asset_index.py created with envelope format, atomic writes, lock, and delegated build_index + + + + Task 3: Write tests for asset_index.py core functionality + tests/test_asset_index.py + + + @paperforge/worker/asset_index.py (just created — full file) + @paperforge/worker/sync.py (lines 1669-1792) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-01..D-07) + + + + - Test 1: get_index_path returns Path ending in "indexes/formal-library.json" + - Test 2: build_envelope wraps items list with schema_version="2", generated_at, paper_count + - Test 3: atomic_write_index writes valid JSON file at target path with correct content + - Test 4: atomic_write_index with concurrent lock times out cleanly (use threading to simulate) + - Test 5: build_index with empty export dir returns 0 and writes empty envelope + - Test 6: build_index delegates to the same code path as old run_index_refresh (structural equivalence on a minimal fixture) + - Test 7: Envelope paper_count matches len(items) + + + + Create `tests/test_asset_index.py` with tests for the core functions. Use the existing test patterns from `tests/test_selection_sync_pdf.py` (pytest fixtures, vault mocking). + + Steps: + 1. Create the test file with these test functions: + - `test_get_index_path_returns_path` — verify get_index_path returns a Path ending with formal-library.json + - `test_build_envelope_empty` — envelope with empty list has schema_version="2", paper_count=0 + - `test_build_envelope_with_items` — envelope with 3 items has paper_count=3, items match input + - `test_atomic_write_index_creates_file` — write to temp dir, verify JSON file exists with correct content + - `test_atomic_write_index_is_atomic` — simulate interrupt (mock NamedTemporaryFile to fail after write), verify no partial file at target + - `test_atomic_write_index_lock_timeout` — acquire lock in thread, then try from main thread, verify filelock.Timeout is raised + - `test_build_index_empty_exports` — mock empty exports dir, verify returns 0 and empty envelope written + - `test_envelope_paper_count_matches` — verify envelope.paper_count always equals len(envelope.items) + + 2. Run tests to confirm they pass: + ```bash + pytest tests/test_asset_index.py -v --tb=short + ``` + + Note: Do NOT attempt to run the full test suite (tests may have pre-existing failures). Only the new test file matters. + + + + pytest tests/test_asset_index.py -v --tb=short 2>&1 + + + + - `tests/test_asset_index.py` exists with at least 6 test functions + - `pytest tests/test_asset_index.py -v --tb=short` exits 0 + - Tests cover: get_index_path, build_envelope, atomic_write, lock behavior, empty-exports edge case + - No pre-existing tests are broken (verify by running only the new test file) + + + New test file passes all asset_index unit tests + + + + + +1. `python -c "from paperforge.worker.asset_index import build_index, get_index_path, atomic_write_index; print('OK')"` — all three exports available +2. `pytest tests/test_asset_index.py -v --tb=short` — all tests pass +3. `grep -n "filelock" pyproject.toml` — dependency added +4. `grep -n "asset_index.build_index" paperforge/worker/sync.py` — delegation in place +5. `grep -n "tempfile.NamedTemporaryFile" paperforge/worker/asset_index.py` — atomic write uses tempfile +6. `grep -n "filelock.FileLock\|filelock" paperforge/worker/asset_index.py` — lock implemented + + + +- asset_index.py module exists with build_index, get_index_path, atomic_write_index exports +- Index writes are atomic (tempfile + os.replace) and locked (filelock, 10s timeout) +- run_index_refresh in sync.py delegates to build_index +- filelock dependency in pyproject.toml +- Tests pass for core functionality (envelope, atomic write, lock, empty exports) + + + +After completion, create `.planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-SUMMARY.md` + diff --git a/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-02-PLAN.md b/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-02-PLAN.md new file mode 100644 index 00000000..d71e48e1 --- /dev/null +++ b/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-02-PLAN.md @@ -0,0 +1,315 @@ +--- +phase: 23-canonical-asset-index-safe-rebuilds +plan: 02 +type: execute +wave: 2 +depends_on: + - 23-01 +files_modified: + - paperforge/worker/asset_index.py + - paperforge/worker/sync.py + - paperforge/cli.py + - tests/test_asset_index.py +autonomous: true +requirements: + - ASSET-01 + - ASSET-02 + - ASSET-03 + - ASSET-04 + +must_haves: + truths: + - "Legacy bare-list formal-library.json is detected on read, backed up as .bak, and transparently migrated to envelope format" + - "refresh_index_entry(vault, key) reads existing index, updates the single matching entry, writes back atomically" + - "run_index_refresh detects schema_version mismatch and auto-triggers full rebuild instead of incremental" + - "Workspace path fields (paper_root, main_note_path, fulltext_path, deep_reading_path, ai_path) appear in each index entry" + - "Path fields are mirrored in the formal note frontmatter for Base view compatibility" + - "--rebuild-index flag on `paperforge sync` forces full rebuild" + artifacts: + - path: "paperforge/worker/asset_index.py" + provides: "refresh_index_entry, legacy migration, workspace paths" + exports: "refresh_index_entry, read_index" + - path: "paperforge/worker/sync.py" + provides: "--rebuild-index flag handling, schema_version check" + - path: "paperforge/cli.py" + provides: "--rebuild-index CLI argument" + - path: "tests/test_asset_index.py" + provides: "Tests for incremental refresh, migration, workspace paths" + min_lines: 150 + key_links: + - from: "asset_index.py :: build_index()" + to: "index envelope :: schema_version" + via: "auto-detect legacy: no schema_version key -> migrate before rebuild" + - from: "asset_index.py :: refresh_index_entry(vault, key)" + to: "index file :: items[]" + via: "match zotero_key, rebuild single entry in-place, atomic write" + - from: "sync.py :: run_index_refresh()" + to: "asset_index.py :: refresh_index_entry()" + via: "incremental path when selected_keys is provided" + - from: "cli.py :: sync command" + to: "sync.py :: run_index_refresh(rebuild=True)" + via: "--rebuild-index flag" + +--- + + +Add legacy format migration, incremental refresh by key, workspace path fields, and the --rebuild-index CLI flag to the asset index system. + +Purpose: Users with existing bare-list formal-library.json files get transparent migration. After sync/ocr/deep-reading, only changed entries are refreshed (faster, safer). Workspace-style path fields (matching Phase 22 decisions) are added to each entry and mirrored in note frontmatter for Base view consumption. + +Output: Extended `asset_index.py` with `refresh_index_entry()`, `read_index()`, legacy migration, workspace path fields + cli.py `--rebuild-index` flag + tests. + + + + +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md + + + +@.planning/STATE.md +@.planning/ROADMAP.md (Phase 23) +@.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-04, D-08..D-14, D-18..D-19) +@.planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-PLAN.md (existing asset_index.py module) +@paperforge/worker/asset_index.py (just created in Plan 01) +@paperforge/worker/sync.py (lines 1669-1792 -- current run_index_refresh) +@paperforge/worker/_utils.py (lines 241-265 -- pipeline_paths paths) +@paperforge/cli.py (sync command registration) +@paperforge/config.py (lines 105-112 -- get_paperforge_schema_version) + + + + + + Task 1: Legacy format detection, backup, and auto-migration in build_index + paperforge/worker/asset_index.py + + + @paperforge/worker/asset_index.py (full file — Plan 01 output) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-04, D-18, D-19) + + + + Add the following functions to `paperforge/worker/asset_index.py`: + + **Function: `read_index(vault: Path) -> dict | list | None`** + - Returns `None` if the index file does not exist + - Reads and parses the JSON at `get_index_path(vault)` + - Returns the raw parsed data (could be dict with envelope, or list for old bare-list format) + + **Function: `is_legacy_format(data) -> bool`** + - Returns `True` if `data` is a `list` (old bare-list format before v1.6) + - Returns `False` if `data` is a `dict` with `"schema_version"` key + + **Function: `migrate_legacy_index(vault: Path) -> bool`** + - Called by `build_index()` before the rebuild loop + - Reads the existing index via `read_index()` + - If the format is legacy (list): + 1. Back up the legacy file: copy to `formal-library.json.bak` (same directory, same name + `.bak`) + 2. Log: `f"Legacy format detected at {path}, backed up to {path}.bak"` + 3. Return True (migration pending — will be overwritten by build_index) + - If already envelope format, return False (no-op) + - If file doesn't exist, return False (fresh build) + - If file doesn't exist or is already envelope: no action, return False + + **Modify `build_index()`:** + - At the start of `build_index()` (after `ensure_base_views` but before the export loop), call: + ```python + migrated = migrate_legacy_index(vault) + if migrated: + print("Legacy index format detected and backed up. Rebuilding with envelope...") + ``` + - Otherwise the rebuild loop naturally writes the new envelope format + + **Edge cases:** + - No existing index file: start fresh (no backup needed, no migration) + - Already envelope format: no action + - Backup file already exists: overwrite with current content (idempotent — last backup is most recent state) + - Corrupt JSON: treat as missing file (start fresh, log warning) + + + + python -c "from paperforge.worker.asset_index import read_index, is_legacy_format, migrate_legacy_index; print('OK')" + + + + - `read_index`, `is_legacy_format`, `migrate_legacy_index` are exported from asset_index + - `is_legacy_format([])` returns True (list = legacy) + - `is_legacy_format({"schema_version": "2", "items": []})` returns False + - `migrate_legacy_index` creates `.bak` copy when legacy format detected (verified via manual test) + - `build_index()` calls `migrate_legacy_index()` at start (grep for call in build_index) + + + Legacy format detection, backup, and auto-migration added to asset_index.py + + + + Task 2: Implement incremental refresh (refresh_index_entry) and schema_version check + paperforge/worker/asset_index.py + + + @paperforge/worker/asset_index.py (full file — includes Task 1 additions) + @paperforge/worker/sync.py (lines 1686-1744 — entry construction logic, shared with build_index) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-08..D-11) + + + + Add the following function to `paperforge/worker/asset_index.py`: + + **Function: `refresh_index_entry(vault: Path, key: str) -> bool`** + - Reads the existing index via `read_index()` + - If the index is in legacy format, calls `build_index(vault)` for full rebuild (returns False to indicate full rebuild happened) + - If the index is envelope format: + 1. Reads the `items` list + 2. Finds the entry where `item["zotero_key"] == key` + 3. If found: rebuilds that single entry (using the same entry-construction logic as `build_index()` — shared helper), replaces it in the items list, preserves all other entries unchanged + 4. If not found: could be a new paper; rebuilds the entry and appends it + 5. Calls `atomic_write_index(path, build_envelope(items))` to write back + 6. Returns True (incremental refresh done) + - **Shared helper approach**: Extract the entry-construction code from `build_index()` into a helper function: + ```python + def _build_entry(item: dict, vault: Path, paths: dict, **kwargs) -> dict: + """Construct a single index entry from a BBT export item.""" + ``` + This is used by both `build_index()` (loop) and `refresh_index_entry()` (single). + - The helper `_build_entry` should accept the same parameters and return the same dict shape as the current entry construction in `build_index()` + - The `key` parameter is the Zotero citation key (8-char alphanumeric) + + **Add to `build_index()`:** + - At the start, add a `schema_version` check: + ```python + # Check if existing index has matching schema_version + existing = read_index(vault) + if isinstance(existing, dict) and existing.get("schema_version") != CURRENT_SCHEMA_VERSION: + print(f"Schema version mismatch: index has {existing.get('schema_version')}, need {CURRENT_SCHEMA_VERSION}. Rebuilding...") + # Continue with full rebuild (no early return) + ``` + + **Edge cases:** + - Index file does not exist yet: cannot refresh — delegate to `build_index()` + - `key` not found in items: rebuild entry from scratch and append (handles newly-synced papers) + - `key` found but export data missing (deleted from Zotero): log warning and skip + - Race condition: lock inside `atomic_write_index` protects concurrent writes + - Empty items list and key not found: add as single entry + + + + python -c "from paperforge.worker.asset_index import refresh_index_entry, _build_entry; print('OK')" + + + + - `refresh_index_entry(vault, key)` is exported from asset_index + - `_build_entry(item, vault, paths)` is a callable helper that returns a dict with same shape as current entry construction + - `refresh_index_entry` handles legacy format by falling back to `build_index` + - `refresh_index_entry` preserves non-matching entries in the items list (only the targeted key changes) + - Schema mismatch check in `build_index` compares existing schema_version vs CURRENT_SCHEMA_VERSION + + + refresh_index_entry and schema_version check implemented + + + + Task 3: Add workspace path fields to index entries, mirror in frontmatter, wire --rebuild-index flag + + paperforge/worker/asset_index.py + paperforge/worker/sync.py + paperforge/cli.py + + + + @paperforge/worker/asset_index.py (full file — includes Tasks 1-2) + @paperforge/worker/sync.py (lines 1711-1740 — current entry dict construction, frontmatter_note function) + @paperforge/cli.py (sync command definition) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-12..D-14) + @.planning/phases/22-configuration-truth-compatibility/22-CONTEXT.md (D-01..D-05 — paper workspace layout) + + + + **A. Add workspace path fields to `_build_entry()` in asset_index.py:** + + After the existing entry fields (zotero_key, domain, title, etc., deep_reading_md_path), add these workspace path fields (per D-12): + + ```python + # Workspace paths (Phase 22 paper workspace structure) + "paper_root": f"Literature/{domain}/{key} - {title_slug}/", + "main_note_path": f"Literature/{domain}/{key} - {title_slug}/{key} - {title_slug}.md", + "fulltext_path": f"Literature/{domain}/{key} - {title_slug}/fulltext.md", + "deep_reading_path": f"Literature/{domain}/{key} - {title_slug}/deep-reading.md", + "ai_path": f"Literature/{domain}/{key} - {title_slug}/ai/", + ``` + + - All paths are relative to vault root, using forward slashes (consistent with existing `note_path` format) + - The `title_slug` comes from `slugify_filename(item["title"])` (already computed in _build_entry) + + **B. Keep existing system paths** (D-13): Do NOT remove `ocr_path`, `meta_path`, `note_path`, `deep_reading_md_path` — these remain for backward compatibility. + + **C. Mirror path fields in note frontmatter** (D-14): + + In the `_build_entry()` function, the existing code writes the formal note via `frontmatter_note()`. The path fields need to be passed through to the note frontmatter as YAML properties: + + - Locate where `frontmatter_note(entry, existing_text)` is called (currently in the `build_index` loop and also in `refresh_index_entry`) + - The `frontmatter_note()` function already generates YAML frontmatter from the entry dict + - Adding the new path fields (`paper_root`, `main_note_path`, etc.) to the entry dict means they automatically become part of the frontmatter that Base views read + - Verify this by examining `frontmatter_note()` in sync.py — it should iterate entry keys for frontmatter fields + + **D. Add --rebuild-index flag to cli.py sync command** (D-10): + + In `paperforge/cli.py`, find the sync command definition and add: + ```python + @click.option("--rebuild-index", is_flag=True, help="Force full rebuild of the canonical asset index") + ``` + + Then in the sync command handler, pass `rebuild_index` to `run_index_refresh()`. + + In `sync.py` `run_index_refresh()`, accept the `rebuild_index: bool = False` parameter: + - If True: skip incremental path, always call `asset_index.build_index(vault, verbose)` + - If False: use existing behavior (currently always full rebuild; will add incremental check later) + + **E. Auto-rebuild on schema_version mismatch** (D-10): + + Already handled in Task 2 within `build_index()` — schema_version mismatch detection triggers full rebuild. + + **Edge cases:** + - `title_slug` may differ from existing directory name if the title changed; this is consistent with current behavior where stale dirs are cleaned up + - Path fields should be consistent across full rebuild and incremental refresh (both use `_build_entry`) + - Frontmatter mirroring: the fields MUST appear in the YAML frontmatter of the formal note so that Base views (which read frontmatter) can use them + + + + python -c "from paperforge.worker.asset_index import _build_entry; e = _build_entry({'key':'abc','title':'Test','domain':'test'}, None, None); assert 'paper_root' in e; assert 'main_note_path' in e; assert 'fulltext_path' in e; assert 'deep_reading_path' in e; assert 'ai_path' in e; print('OK')" + + + + - `_build_entry` result dict contains `paper_root`, `main_note_path`, `fulltext_path`, `deep_reading_path`, `ai_path` keys + - All path values use forward slashes and are relative to vault root + - Existing system paths (`ocr_path`, `meta_path`, `note_path`, `deep_reading_md_path`) are still present (removed in error if missing) + - `frontmatter_note` in sync.py includes the new path fields (check if frontmatter_note uses all entry keys) + - `cli.py` sync command has `--rebuild-index` option + - `sync.py` `run_index_refresh()` accepts `rebuild_index` parameter + + + Workspace path fields added, frontmatter mirrored, --rebuild-index flag wired + + + + + +1. `python -c "from paperforge.worker.asset_index import read_index, is_legacy_format, migrate_legacy_index, refresh_index_entry, _build_entry; print('OK')"` — all new exports available +2. `python -c "e = {'key':'abc','title':'Test','domain':'test'}; from paperforge.worker.asset_index import _build_entry as b; r = b(e, None, None); assert all(k in r for k in ['paper_root','main_note_path','fulltext_path','deep_reading_path','ai_path'])"` — path fields present +3. `grep 'rebuild-index' paperforge/cli.py` — CLI flag defined +4. `grep 'rebuild_index' paperforge/worker/sync.py` — parameter accepted +5. `python -c "from paperforge.worker.asset_index import is_legacy_format; assert is_legacy_format([]) == True; assert is_legacy_format({'schema_version':'2'}) == False"` — legacy detection works +6. `grep -n '\.bak' paperforge/worker/asset_index.py` — migration creates backup + + + +- Legacy bare-list formal-library.json is detected and backed up as .bak before transparent migration +- refresh_index_entry(vault, key) updates single entry atomically +- Workspace path fields (paper_root, main_note_path, fulltext_path, deep_reading_path, ai_path) present in each index entry +- Path fields mirrored in note frontmatter for Base compatibility +- CLI --rebuild-index flag forces full rebuild +- Schema_version mismatch auto-triggers full rebuild + + + +After completion, create `.planning/phases/23-canonical-asset-index-safe-rebuilds/23-02-SUMMARY.md` + diff --git a/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-03-PLAN.md b/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-03-PLAN.md new file mode 100644 index 00000000..5d542634 --- /dev/null +++ b/.planning/phases/23-canonical-asset-index-safe-rebuilds/23-03-PLAN.md @@ -0,0 +1,365 @@ +--- +phase: 23-canonical-asset-index-safe-rebuilds +plan: 03 +type: execute +wave: 3 +depends_on: + - 23-02 +files_modified: + - paperforge/worker/ocr.py + - paperforge/worker/deep_reading.py + - paperforge/worker/repair.py + - paperforge/worker/sync.py + - tests/test_asset_index_integration.py +autonomous: true +requirements: + - ASSET-03 + - ASSET-04 + - MIG-02 + +must_haves: + truths: + - "After OCR completes, incremental refresh is called for the affected zotero_key instead of full rebuild" + - "After deep-reading status sync, incremental refresh is called for the affected zotero_key" + - "After repair operations, incremental refresh is called for affected keys" + - "Incremental refresh is triggered by default; full rebuild only runs when --rebuild-index is passed or schema_version mismatches" + artifacts: + - path: "paperforge/worker/ocr.py" + provides: "Post-OCR incremental refresh" + modifies: "call site _sync.run_index_refresh -> asset_index.refresh_index_entry" + - path: "paperforge/worker/deep_reading.py" + provides: "Post-deep-reading incremental refresh" + adds: "asset_index.refresh_index_entry call after status sync" + - path: "paperforge/worker/repair.py" + provides: "Post-repair incremental refresh" + adds: "asset_index.refresh_index_entry call after repair" + - path: "paperforge/worker/sync.py" + provides: "Default incremental path in run_index_refresh" + - path: "tests/test_asset_index_integration.py" + provides: "Integration tests for full cycle" + min_lines: 80 + key_links: + - from: "ocr.py :: OCR loop end" + to: "asset_index.py :: refresh_index_entry(vault, ocr_key)" + via: "incremental refresh per OCR'd zotero_key" + - from: "deep_reading.py :: run_deep_reading loop" + to: "asset_index.py :: refresh_index_entry(vault, dr_key)" + via: "incremental refresh after status sync per key" + - from: "repair.py :: repair operations" + to: "asset_index.py :: refresh_index_entry(vault, repaired_key)" + via: "incremental refresh after fixing paths/states" + +--- + + +Wire incremental index refresh into all worker operations that modify paper state (OCR, deep-reading, repair) so they update only the affected index entries instead of rebuilding the entire index. Make the default sync path use incremental refresh on individual keys rather than full rebuild. + +Purpose: Workers that modify a single paper's state (OCR completion, deep-reading status sync, path repair) should NOT trigger a complete index rebuild. Incremental refresh by key is faster, safer, and avoids unnecessary writes. + +Output: All four worker call sites updated + integration tests. + + + +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md + + + +@.planning/STATE.md +@.planning/ROADMAP.md (Phase 23) +@.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-11) +@.planning/phases/23-canonical-asset-index-safe-rebuilds/23-01-PLAN.md (asset_index.py — build_index, refresh_index_entry) +@paperforge/worker/asset_index.py (full file — build_index, refresh_index_entry) +@paperforge/worker/ocr.py (line 1813 — current _sync.run_index_refresh(vault) post-OCR) +@paperforge/worker/deep_reading.py (lines 25-137 — run_deep_reading, status sync loop) +@paperforge/worker/repair.py (lines 1-30 — imports, repair operations overview) +@paperforge/worker/sync.py (lines 1669-1792 — run_index_refresh, rebuild_index parameter) + + + + + + Task 1: Switch OCR post-processing from full rebuild to incremental refresh by key + paperforge/worker/ocr.py + + + @paperforge/worker/ocr.py (lines 1806-1817 — the post-OCR sync+index block) + @paperforge/worker/ocr.py (lines 1-30 — existing imports) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-11) + + + + In `paperforge/worker/ocr.py`, modify the post-OCR index refresh block (currently lines 1811-1813): + + **Current code:** + ```python + try: + _sync.run_selection_sync(vault) + _sync.run_index_refresh(vault) + except Exception as e: + logger.error("Post-OCR sync failed: %s", e) + ``` + + **Replace with incremental refresh:** + ```python + from paperforge.worker.asset_index import refresh_index_entry + + try: + _sync.run_selection_sync(vault) + # Incremental refresh: update only the affected paper(s) + # The OCR queue may have processed multiple keys; use the last key from context + # or fall back to full rebuild if no single key is identifiable + ocr_keys = [r.get("key", "") for r in ocr_queue if r.get("queue_status") == "done"] + for ocr_key in ocr_keys: + if ocr_key: + refresh_index_entry(vault, ocr_key) + if verbose: + print(f"ocr: refreshed {len(ocr_keys)} index entries incrementally") + except ImportError: + # Fallback: if asset_index module not available (pre-migration), use full rebuild + _sync.run_index_refresh(vault) + except Exception as e: + logger.error("Post-OCR index refresh failed: %s", e) + ``` + + Add the import at the top of the file (with the other imports). The import is: + ```python + from paperforge.worker.asset_index import refresh_index_entry + ``` + + **Important:** Keep the existing `_sync.run_selection_sync(vault)` call — that still does the library-record sync. Only the index refresh part changes from full rebuild to incremental. + + **Edge cases:** + - OCR queue has no "done" items: no refresh needed (the variable `changed` tracks this — only refresh if `changed > 0`) + - OCR queue is empty: skip refresh entirely + - `refresh_index_entry` handles legacy format detection internally (falls back to full rebuild) + + + + grep -n "refresh_index_entry\|oc: refreshed" paperforge/worker/ocr.py + + + + - `paperforge/worker/ocr.py` imports `refresh_index_entry` from `paperforge.worker.asset_index` + - Post-OCR block calls `refresh_index_entry(vault, ocr_key)` instead of `_sync.run_index_refresh(vault)` + - Fallback to full rebuild exists for pre-migration state (ImportError catch) + + + OCR post-processing uses incremental refresh by key instead of full rebuild + + + + Task 2: Add incremental refresh to deep-reading status sync + paperforge/worker/deep_reading.py + + + @paperforge/worker/deep_reading.py (lines 25-137 — run_deep_reading function) + @paperforge/worker/deep_reading.py (lines 1-17 — existing imports) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-11) + + + + In `paperforge/worker/deep_reading.py`, add incremental index refresh after the library-record status sync loop. + + 1. Add import at the top: + ```python + from paperforge.worker.asset_index import refresh_index_entry + ``` + + 2. After the loop that syncs `deep_reading_status` in library records (lines 56-66, where `record_path.write_text(...)` is called), add an incremental refresh for each synced key: + ```python + # Refresh canonical index for each synced paper + for record in records: + key = record["zotero_key"] + if key: + try: + refresh_index_entry(vault, key) + except Exception: + logger.warning("Failed to refresh index entry for %s", key) + ``` + + Place this block right after the `synced += 1` loop completes and the report lines are written (after line 79, before the pending_queue reporting block). + + **Important:** Keep the `synced` counter for progress reporting. The index refresh should happen even when `synced == 0` if there are changes in the formal note content that affect the index (e.g., deep_reading_path field). + + **Edge cases:** + - deep-reading runs but nothing changed (`synced == 0`): still refresh for papers that have pending status but were already up-to-date (needed for deep_reading_path field) + - `refresh_index_entry` raises: log warning, do not abort the deep-reading worker + - Multiple papers synced: refresh each one incrementally + + + + grep -n "refresh_index_entry" paperforge/worker/deep_reading.py + + + + - `paperforge/worker/deep_reading.py` imports `refresh_index_entry` from `paperforge.worker.asset_index` + - After the deep-reading status sync loop, `refresh_index_entry` is called for each paper + - Failure to refresh logs warning but does not abort + + + Deep-reading post-processing calls incremental refresh for each synced paper + + + + Task 3: Add incremental refresh to repair operations and verify default sync behavior + + paperforge/worker/repair.py + paperforge/worker/sync.py + + + + @paperforge/worker/repair.py (lines 1-60 — imports, repair function signatures) + @paperforge/worker/sync.py (lines 1669-1746 — run_index_refresh, rebuild_index parameter) + @.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md (D-11) + + + + **A. Add incremental refresh to repair.py:** + + In `paperforge/worker/repair.py`: + 1. Add import: `from paperforge.worker.asset_index import refresh_index_entry` + 2. Find where repair fixes paths or states for individual papers. The repair operations that modify paper state should trigger incremental refresh. + 3. After each repair action that affects a specific paper (e.g., `--fix-paths` or status repair), call: + ```python + try: + refresh_index_entry(vault, repaired_key) + except Exception as e: + logger.warning("Failed to refresh index for %s: %s", repaired_key, e) + ``` + + **B. Verify sync.py default behavior:** + + In `paperforge/worker/sync.py`, verify the `run_index_refresh()` function: + - The `rebuild_index` parameter (added in Plan 02) controls whether to call `asset_index.build_index()` (full rebuild) or something else + - Currently `run_index_refresh` always calls `asset_index.build_index()` for the full rebuild path + - Keep this behavior: when called without args (from CLI without `--rebuild-index`), it still does a full rebuild by default because selection-sync may affect many papers + - The incremental path is triggered from OCR, deep-reading, and repair (individual paper changes) + - Add a docstring to `run_index_refresh()` explaining this convention: + ```python + """Refresh the canonical asset index. + + Default behavior: full rebuild. This is the safe default because + selection-sync may affect many papers. Workers that modify individual + papers (ocr, deep-reading, repair) use asset_index.refresh_index_entry() + for incremental refresh by key. + + Args: + vault: Path to the vault root. + verbose: If True, print detailed progress. + rebuild_index: If True, force full rebuild (default: True for sync). + """ + ``` + + **Edge cases:** + - Repair with no specific paper (library-wide operations): skip incremental refresh, no action needed + - Sync with `--rebuild-index`: forced full rebuild regardless of current state + - Sync without `--rebuild-index`: still does full rebuild (safe default for multi-paper changes) + + + + grep -n "refresh_index_entry" paperforge/worker/repair.py; grep -n "def run_index_refresh" paperforge/worker/sync.py + + + + - `paperforge/worker/repair.py` imports `refresh_index_entry` + - Per-paper repair operations call `refresh_index_entry` after fixing + - `run_index_refresh` in sync.py has docstring explaining full-rebuild default convention + + + Repair wired to incremental refresh, sync.py convention documented + + + + Task 4: Integration tests for incremental refresh across worker operations + tests/test_asset_index_integration.py + + + @paperforge/worker/asset_index.py (full file) + @paperforge/worker/ocr.py (modified — post-OCR incremental refresh) + @paperforge/worker/deep_reading.py (modified — post-deep-reading incremental refresh) + @tests/test_asset_index.py (existing unit tests) + + + + Create `tests/test_asset_index_integration.py` with integration-level tests: + + Test functions: + + ```python + def test_incremental_refresh_preserves_unrelated_entries(): + """refresh_index_entry only modifies the targeted entry's fields.""" + # Setup: create a test index with 3 entries + # Call refresh_index_entry for key of entry 1 + # Read back index + # Assert entries 2 and 3 have identical content (no side effects) + pass + + def test_incremental_refresh_adds_new_key(): + """refresh_index_entry appends a new key not yet in the index.""" + pass + + def test_incremental_refresh_fallback_on_legacy(): + """refresh_index_entry falls back to build_index when index is legacy format.""" + pass + + def test_ocr_calls_incremental_not_full(): + """Verify that OCR post-processing calls refresh_index_entry, not run_index_refresh.""" + # Structural check: grep ocr.py for the call pattern + pass + + def test_deep_reading_calls_incremental(): + """Verify that deep_reading.py calls refresh_index_entry.""" + pass + + def test_workspace_paths_in_entry(): + """Each entry in the built index contains all 5 workspace path fields.""" + pass + + def test_path_fields_consistent_after_incremental(): + """After incremental refresh, the targeted entry has consistent path fields.""" + pass + ``` + + Run the tests: + ```bash + pytest tests/test_asset_index_integration.py -v --tb=short 2>&1 + ``` + If any test fixture depends on a real vault, use unittest.mock to mock `paperforge_paths` and file I/O. + + **Important:** These are structural/integration tests that verify call sites and data consistency. They should use mocking to avoid depending on real vault fixtures. + + + + pytest tests/test_asset_index_integration.py -v --tb=short 2>&1 + + + + - `tests/test_asset_index_integration.py` exists with at least 5 test functions + - `pytest tests/test_asset_index_integration.py -v --tb=short` exits 0 + - Tests cover: incremental preserves unrelated entries, fallback on legacy, workspace paths, and structural verification of OCR/deep-reading call sites + + + Integration tests pass for incremental refresh across all worker operations + + + + + +1. `grep -n "refresh_index_entry" paperforge/worker/ocr.py` — OCR wired +2. `grep -n "refresh_index_entry" paperforge/worker/deep_reading.py` — deep-reading wired +3. `grep -n "refresh_index_entry" paperforge/worker/repair.py` — repair wired +4. `pytest tests/test_asset_index_integration.py -v --tb=short 2>&1` — integration tests pass +5. `python -c "from paperforge.worker.asset_index import refresh_index_entry; print('OK')"` — module importable + + + +- OCR post-processing calls refresh_index_entry for each completed key instead of full rebuild +- Deep-reading status sync triggers incremental refresh for each synced paper +- Repair triggers incremental refresh for each repaired paper +- run_index_refresh() has docstring clarifying full-rebuild convention +- Integration tests verify call sites and data consistency + + + +After completion, create `.planning/phases/23-canonical-asset-index-safe-rebuilds/23-03-SUMMARY.md` + \ No newline at end of file diff --git a/.planning/phases/24-derived-lifecycle-health-maturity/24-02-SUMMARY.md b/.planning/phases/24-derived-lifecycle-health-maturity/24-02-SUMMARY.md new file mode 100644 index 00000000..b29008e3 --- /dev/null +++ b/.planning/phases/24-derived-lifecycle-health-maturity/24-02-SUMMARY.md @@ -0,0 +1,100 @@ +--- +phase: 24-derived-lifecycle-health-maturity +plan: 02 +subsystem: worker +tags: [asset-state, lifecycle, health, maturity, next-step, index] + +# Dependency graph +requires: + - phase: 24-01 + provides: "compute_lifecycle, compute_health, compute_maturity, compute_next_step in asset_state.py" +provides: + - "Every canonical index entry now carries embedded lifecycle, health, maturity, and next_step fields derived from source artifacts" + - "Both build_index() and refresh_index_entry() code paths produce entries with the four new fields" +affects: [25-surface-convergence, 26-ai-context-packs, plugin-dashboard] + +# Tech tracking +tech-stack: + added: [] + patterns: + - "Lazy import pattern: asset_state functions imported inside _build_entry() to avoid circular dependencies with sync.py" + - "Pure derivation: all four fields computed from the entry dict itself, no filesystem access, no side effects" + +key-files: + created: [] + modified: + - "paperforge/worker/asset_index.py - Added import and 4 field assignments to _build_entry()" + - "tests/test_asset_index_integration.py - Added TestDerivedStateFields class (6 tests)" + +key-decisions: + - "Derived fields inserted after entry dict construction but before formal note write — ensures note frontmatter does not include machine-only fields" + - "Lazy import inside _build_entry() follows the existing pattern established for sync.py/ocr.py imports — no new circular dependency risk" + - "Call order: lifecycle first, then health, then maturity (calls lifecycle internally), then next_step — ensures maturity has correct lifecycle value" + +patterns-established: + - "Pattern: Derived machine-only fields appended to entry dict after all source-artifact fields, before formal note write — keeps machine state out of user-visible notes" + +requirements-completed: [STATE-01, STATE-02, STATE-03, STATE-04, AIC-01] + +# Metrics +duration: 25min +completed: 2026-05-04 +--- + +# Phase 24 Plan 02: asset_state Integration into asset_index Summary + +**All four compute functions from asset_state.py wired into _build_entry() — every canonical index entry now carries embedded lifecycle, health, maturity, and next_step fields derived from source artifacts** + +## Performance + +- **Duration:** 25 min +- **Started:** 2026-05-04T02:40:21Z +- **Completed:** 2026-05-04T03:04:55Z +- **Tasks:** 3 +- **Files modified:** 2 + +## Accomplishments +- Imported `compute_lifecycle`, `compute_health`, `compute_maturity`, `compute_next_step` from `asset_state` into `_build_entry()` +- Added 4 field assignments after entry dict construction, before formal note write — fields flow automatically through both `build_index()` and `refresh_index_entry()` +- 6 new integration tests verify field presence, type correctness, and value validity across both full-build and incremental-refresh paths +- All 55 asset-specific tests pass (23 asset_index + 26 asset_state + 6 new integration) + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Add derived state fields to _build_entry()** - `8fd963f` (feat) +2. **Task 2: Add integration tests for derived state fields** - `043df53` (test) +3. **Task 3: Verify full test suite and serialization** - No code changes (verification-only) + +## Files Created/Modified +- `paperforge/worker/asset_index.py` - Added lazy import of 4 asset_state functions; added 4 field assignments (lifecycle, health, maturity, next_step) after entry dict construction +- `tests/test_asset_index_integration.py` - Added `TestDerivedStateFields` class with 6 test methods covering field presence, type validation, and value correctness + +## Decisions Made +- Lazy import inside `_build_entry()` follows existing pattern (sync.py, ocr.py imports) — avoids circular dependency risk +- Derived fields inserted after entry dict closing `}` but before `# Write / update the formal note` — keeps machine-only fields out of user-visible Obsidian note frontmatter +- Call order: lifecycle → health → maturity → next_step. Maturity calls `compute_lifecycle()` internally, so lifecycle must be set first + +## Deviations from Plan + +None — plan executed exactly as written. + +The 3 pre-existing test failures (`test_doctor_on_empty_vault`, `test_corrupt_meta_json_does_not_crash`, `test_request_timeout_triggers_retry_not_fatal`) are confirmed unrelated to these changes (verified by git stash revert). + +## Issues Encountered + +None. + +## User Setup Required + +None — no external service configuration required. + +## Next Phase Readiness +- Phase 25 (surface convergence): Ready — canonical index entries now carry all four derived state fields needed by status dashboards and plugin surfaces +- Phase 26 (AI context packs): Ready — lifecycle and maturity fields available for context-pack quality gates +- No blockers + +--- +*Phase: 24-derived-lifecycle-health-maturity* +*Completed: 2026-05-04* diff --git a/.planning/phases/25-surface-convergence-doctor-repair/25-CONTEXT.md b/.planning/phases/25-surface-convergence-doctor-repair/25-CONTEXT.md new file mode 100644 index 00000000..db224a4e --- /dev/null +++ b/.planning/phases/25-surface-convergence-doctor-repair/25-CONTEXT.md @@ -0,0 +1,109 @@ +# Phase 25: Surface Convergence, Doctor & Repair - Context + +**Gathered:** 2026-05-04 +**Status:** Ready for planning + + +## Phase Boundary + +Surface convergence: make `paperforge status`, plugin dashboard, Base views, doctor, and repair all consume the same lifecycle/health/maturity data from the canonical index. No more independent computation across surfaces. + +Does NOT cover AI context packs (Phase 26) or the LLMWiki concept network (v1.7). + + + + +## Implementation Decisions + +### status --json output +- **D-01:** `paperforge status --json` reads from canonical index instead of filesystem scanning. +- **D-02:** Output includes: paper_count, lifecycle_level_counts, health_aggregate (4 dimensions with count per status), maturity_distribution. +- **D-03:** Legacy `run_status()` in `status.py` delegates to `asset_index.py` + `asset_state.py` for derived data. +- **D-04:** Filesystem scan is retained as fallback when canonical index is unavailable. + +### Plugin Dashboard +- **D-05:** Plugin reads `formal-library.json` directly via `readFileSync()` instead of spawning `python -m paperforge status --json`. +- **D-06:** Only the envelope metadata and summary fields are read (not full items list) — keeps dashboard fast. +- **D-07:** Existing `_fetchStats()` method refactored to read the JSON file directly, falling back to Python CLI if file is missing. + +### Doctor +- **D-08:** New "Index Health" section in `paperforge doctor` output, aggregating from canonical index health fields. +- **D-09:** Shows: PDF Health (OK/warn/fail), OCR Health (OK/warn/fail), Note Health, Asset Health counts. +- **D-10:** Existing checks unchanged — this is additive. + +### Repair +- **D-11:** Repair keeps source-first + rebuild pattern: fix source artifacts, then rebuild canonical index. Never edits index directly. +- **D-12:** After repair, `repair.py` calls `rebuild_index()` to regenerate the canonical index. + +### Base views +- **D-13:** Base `build_base_views()` adds lifecycle, maturity_level, and next_step columns. +- **D-14:** Old `has_pdf`, `do_ocr`, `analyze`, `ocr_status` columns removed from Base views (superseded by lifecycle). +- **D-15:** Sort order: lifecycle ascending (show auto-computed readiness, not raw fields). +- **D-16:** Base filters updated to use lifecycle instead of raw status combinations. + +### the agent's Discretion +- Exact format of plugin dashboard summary (how much of the index to cache in memory) +- Column names in Base views (Chinese labels: 生命周期/就绪状态 等) +- Whether Doctor Index Health is a separate section or merged into existing checks + + + + +## Canonical References + +### Phase scope and requirements +- `.planning/ROADMAP.md` §Phase 25 — Goal: "make status, dashboard, Bases, doctor, repair consume same canonical semantics" +- `.planning/REQUIREMENTS.md` — SURF-01..04, MIG-01, MIG-03, MIG-04 +- `.planning/phases/22-configuration-truth-compatibility/22-CONTEXT.md` — Plugin config truth (reads paperforge.json) +- `.planning/phases/23-canonical-asset-index-safe-rebuilds/23-CONTEXT.md` — Canonical index format +- `.planning/phases/24-derived-lifecycle-health-maturity/24-CONTEXT.md` — lifecycle/health/maturity/next_step fields + +### Source code +- `paperforge/worker/status.py` — Current `run_status()` and `run_doctor()` **Primary refactoring target** +- `paperforge/plugin/main.js` — Current dashboard, `_fetchStats()` (line 261), `_renderStats()` (line 282) +- `paperforge/worker/base_views.py` — `build_base_views()` (line 44), `ensure_base_views()` (line 378) +- `paperforge/worker/repair.py` — `run_repair()` (line 173) +- `paperforge/worker/asset_index.py` — `build_index()`, `refresh_index_entry()` +- `paperforge/worker/asset_state.py` — `compute_lifecycle()`, `compute_health()`, `compute_maturity()`, `compute_next_step()` + + + + +## Existing Code Insights + +### Reusable Assets +- `paperforge/worker/asset_index.py` `build_index()` — Can be called after repair to regenerate canonical index +- `paperforge/worker/asset_state.py` — All four derivation functions ready to consume +- `paperforge/worker/status.py:404-528` — Current `run_status()` — full rewrite target +- `paperforge/plugin/main.js:261-278` — Current `_fetchStats()` — refactor to direct JSON read + +### Established Patterns +- Plugin reads `paperforge.json` directly (Phase 22). Same pattern for `formal-library.json`. +- Base views are Python-generated `.base` files — fully under `build_base_views()` control. +- Doctor checks are additive in `run_doctor()`: "Index Health" as new section. + +### Integration Points +- `paperforge/worker/status.py:404` — `run_status()` — change from filesystem to index-based counting +- `paperforge/plugin/main.js:261` — `_fetchStats()` — change from CLI spawn to JSON file read +- `paperforge/worker/base_views.py:44` — `build_base_views()` — replace columns +- `paperforge/worker/repair.py:173` — `run_repair()` — add `build_index()` call after fix + + + + +## Specific Ideas + +- Plugin should watch `formal-library.json` for changes and auto-refresh dashboard (optional, can defer if too complex). +- Status --json should use the canonical index summary so that `paperforge status` is instant even on large libraries. +- Base views should preserve user customization via the PAPERFORGE_VIEW_PREFIX merge logic already in place. + + + + +None — discussion stayed within phase scope + + +--- + +*Phase: 25-surface-convergence-doctor-repair* +*Context gathered: 2026-05-04* diff --git a/.planning/phases/26-traceable-ai-context-packs/26-01-PLAN.md b/.planning/phases/26-traceable-ai-context-packs/26-01-PLAN.md new file mode 100644 index 00000000..e3e10dad --- /dev/null +++ b/.planning/phases/26-traceable-ai-context-packs/26-01-PLAN.md @@ -0,0 +1,409 @@ +--- +phase: 26-traceable-ai-context-packs +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - paperforge/worker/sync.py (add migrate_to_workspace function) + - paperforge/worker/asset_index.py (update _build_entry to write to workspace paths) + - tests/test_migration.py +autonomous: true +requirements: + - AIC-02 + - AIC-03 + - AIC-04 + +must_haves: + truths: + - "Existing flat Literature// - .md notes are copied into workspace directories on first sync" + - "## 🔍 精读 section is extracted from the main note and written as separate deep-reading.md in the workspace" + - "Migration is idempotent: already-migrated papers are skipped on re-sync" + - "The original flat note file is preserved (copy-not-move)" + - "Canonical index entries' note_path fields point to the new workspace location" + - "New papers (first sync without a prior flat note) create workspace structure directly" + - "ai/ directory is created inside each paper workspace" + artifacts: + - path: paperforge/worker/sync.py + provides: "migrate_to_workspace() function — copies flat notes to workspace dir, splits deep-reading" + - path: paperforge/worker/asset_index.py + provides: "Updated _build_entry writes notes to workspace path instead of flat path" + - path: tests/test_migration.py + provides: "Tests for migration function, _build_entry workspace write, and idempotency" + key_links: + - from: "sync.py:run_index_refresh" + to: "sync.py:migrate_to_workspace" + via: "call migrate_to_workspace() before build_index() in run_index_refresh" + pattern: "migrate_to_workspace" + - from: "asset_index.py:_build_entry" + to: "workspace main_note_path" + via: "write note to Literature/<domain>/<key> - <slug>/<key> - <slug>.md with ai/ dir" + pattern: "main_note_path|paper_root" +--- + +<objective> +Implement flat-to-workspace note migration so that the canonical index's declared workspace paths (`paper_root`, `main_note_path`, `deep_reading_path`, `ai_path`) point to real files instead of hypothetical locations. This migration is a prerequisite for AI context packs (AIC-02, AIC-03, AIC-04) because the context pack must reference workspace-structured assets. + +Purpose: Existing Phase 22-25 code declares workspace paths in the canonical index entry (e.g. `Literature/<domain>/<key> - <slug>/<key> - <slug>.md`) but `_build_entry` still writes notes to the flat path (`Literature/<domain>/<key> - <slug>.md`). Without migration, the workspace paths declared in the index point to nonexistent files, breaking traceability for AI context packs. + +Output: +- `paperforge/worker/sync.py` — new `migrate_to_workspace()` function +- `paperforge/worker/asset_index.py` — updated `_build_entry()` writes to workspace path +- `tests/test_migration.py` — migration tests +</objective> + +<execution_context> +@/C/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@/C/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> +@.planning/STATE.md +@.planning/ROADMAP.md +@.planning/phases/26-traceable-ai-context-packs/26-CONTEXT.md + +<interfaces> +From paperforge/worker/asset_index.py — current _build_entry pattern: +```python +# Line 253 — note path is always flat: +note_path = paths["literature"] / domain / f"{key} - {title_slug}.md" +# Line 296-301 — workspace paths declared in entry but never actually created: +"paper_root": f"Literature/{domain}/{key} - {title_slug}/", +"main_note_path": f"Literature/{domain}/{key} - {title_slug}/{key} - {title_slug}.md", +"fulltext_path": f"Literature/{domain}/{key} - {title_slug}/fulltext.md", +"deep_reading_path": f"Literature/{domain}/{key} - {title_slug}/deep-reading.md", +"ai_path": f"Literature/{domain}/{key} - {title_slug}/ai/", +``` + +From paperforge/worker/sync.py — deep-reading extraction helper: +```python +# Line 564 +DEEP_READING_HEADER = "## 🔍 精读" +# Line 567 +def extract_preserved_deep_reading(text: str) -> str: + # Extracts ## 🔍 精读 section via regex + match = re.search("^## 🔍 精读\\s*$", text, re.MULTILINE) + if not match: return "" + return text[match.start():].strip() +# Line 583 +def has_deep_reading_content(text: str) -> bool: + # Returns True if deep-reading section has substantive content (not just scaffolds) +``` + +From paperforge/worker/sync.py — run_index_refresh pattern: +```python +# Line ~? — current sync flow +def run_index_refresh(vault, verbose=False, rebuild_index=False) -> int: + # calls build_index(vault, verbose) or refresh_index_entry +``` + +From paperforge/worker/asset_index.py — build_index pattern: +```python +# Line 322 +def build_index(vault: Path, verbose: bool = False) -> int: + # Iterates export JSONs, calls _build_entry for each item + # Writes envelope via atomic_write_index +``` + +From paperforge/worker/asset_index.py — _build_entry writes note at line 310-312: +```python +note_path.parent.mkdir(parents=True, exist_ok=True) +existing_text = note_path.read_text(encoding="utf-8") if note_path.exists() else "" +note_path.write_text(frontmatter_note(entry, existing_text), encoding="utf-8") +``` + +From paperforge/worker/sync.py — frontmatter_note signatures (line 1598): +```python +def frontmatter_note(entry: dict, existing_text: str = "") -> str: + # Generates full markdown note from entry dict + preserves deep-reading if present +``` +</interfaces> +</context> + +<tasks> + +<task type="auto" tdd="false"> +<name>Task 1: Add migrate_to_workspace() and update _build_entry() for workspace path writing</name> +<files> + paperforge/worker/sync.py + paperforge/worker/asset_index.py +</files> + +<read_first> + paperforge/worker/sync.py (add migration function near top of file, before run_index_refresh) + paperforge/worker/asset_index.py:210-314 (current _build_entry implementation — modify note writing logic) +</read_first> + +<action> +Perform two coordinated changes: + +**Change A — Add `migrate_to_workspace(vault: Path, paths: dict) -> int` to sync.py:** + +This function scans the canonical index and migrates any paper whose flat note exists but workspace directory does not. It is called by `run_index_refresh()` BEFORE `build_index()`. + +```python +def migrate_to_workspace(vault: Path, paths: dict) -> int: + """Migrate flat literature notes into paper workspace directories. + + Copies each flat note at Literature/<domain>/<key> - <Title>.md into: + Literature/<domain>/<key> - <Title>/<key> - <Title>.md + Extracts ## 🔍 精读 into: + Literature/<domain>/<key> - <Title>/deep-reading.md + Creates: + Literature/<domain>/<key> - <Title>/ai/ (empty directory) + + Idempotent: skips papers whose workspace directory already exists. + The original flat note is preserved (copy-not-move per D-12). + + Returns: number of papers migrated (0 means all are already workspace). + """ + from paperforge.worker.asset_index import read_index, get_index_path + from paperforge.worker.sync import extract_preserved_deep_reading + + index = read_index(vault) + if index is None: + return 0 # No index yet — nothing to migrate + + items = index.get("items", []) if isinstance(index, dict) else [] + migrated = 0 + + for entry in items: + key = entry.get("zotero_key", "") + domain = entry.get("domain", "") + title = entry.get("title", "") + if not key or not domain: + continue + + # Compute paths using the same slugify as _build_entry + from paperforge.worker._utils import slugify_filename + title_slug = slugify_filename(title) + workspace_dir = paths["literature"] / domain / f"{key} - {title_slug}" + + # Skip if already migrated + if workspace_dir.exists(): + continue + + # Find the flat note path + flat_note_path = paths["literature"] / domain / f"{key} - {title_slug}.md" + if not flat_note_path.exists(): + # Try other potential flat note paths (e.g. slightly different slug) + candidates = list((paths["literature"] / domain).glob(f"{key} - *.md")) + if candidates: + flat_note_path = candidates[0] + else: + continue # No flat note to migrate + + # Create workspace directory + workspace_dir.mkdir(parents=True, exist_ok=True) + + # Read flat note content + content = flat_note_path.read_text(encoding="utf-8") + + # Write main note to workspace (copy of flat note) + main_note_path = workspace_dir / f"{key} - {title_slug}.md" + main_note_path.write_text(content, encoding="utf-8") + + # Extract deep-reading section and write to separate file + preserved = extract_preserved_deep_reading(content) + if preserved: + deep_reading_path = workspace_dir / "deep-reading.md" + deep_reading_path.write_text(preserved, encoding="utf-8") + + # Create ai/ directory + ai_dir = workspace_dir / "ai" + ai_dir.mkdir(exist_ok=True) + + migrated += 1 + + if migrated > 0: + print(f"migrate_to_workspace: migrated {migrated} paper(s) to workspace structure") + + return migrated +``` + +**Change B — Update `_build_entry` in asset_index.py to write notes to workspace path:** + +Modify the note-writing section of `_build_entry()` (around line 253-312) per D-11 through D-15: + +1. Compute the workspace `main_note_path` using the same formula as the entry's `main_note_path` field: + ```python + workspace_dir = paths["literature"] / domain / f"{key} - {title_slug}" + main_note_path = workspace_dir / f"{key} - {title_slug}.md" + ``` + +2. Determine the write target and existing-text source: + ```python + # If workspace dir exists, read from and write to workspace (migrated or direct) + if workspace_dir.exists(): + existing_text = main_note_path.read_text(encoding="utf-8") if main_note_path.exists() else "" + write_target = main_note_path + # Ensure subdirs exist + for d in [workspace_dir, workspace_dir / "ai"]: + d.mkdir(parents=True, exist_ok=True) + else: + # Legacy: read from flat path, write to flat path (transitional) + existing_text = note_path.read_text(encoding="utf-8") if note_path.exists() else "" + write_target = note_path + note_path.parent.mkdir(parents=True, exist_ok=True) + ``` + +3. Write the note: + ```python + write_target.write_text(frontmatter_note(entry, existing_text), encoding="utf-8") + ``` + +4. Keep the entry's workspace path fields unchanged (they're already correct): + ```python + "paper_root": f"Literature/{domain}/{key} - {title_slug}/", + "main_note_path": f"Literature/{domain}/{key} - {title_slug}/{key} - {title_slug}.md", + # etc. + ``` + +**Change C — Wire migration into `run_index_refresh()` in sync.py:** + +Inside `run_index_refresh()`, BEFORE the `build_index()` call: +```python +from paperforge.worker._utils import pipeline_paths +paths = pipeline_paths(vault) +migrate_to_workspace(vault, paths) # Per D-11: migrate on first sync +``` + +This ensures flat notes are copied to workspace before _build_entry runs, which makes `_build_entry` see the workspace dir and write updates there. + +**Change D — Remove `note_path` field from entry (optional):** +Keep `note_path` as-is for backward compatibility (flat path in entry). The workspace paths (`main_note_path`, `paper_root`) are the canonical paths. `note_path` can be deprecated in a future phase. + +Acceptance criteria: +- `sync.py` contains `def migrate_to_workspace(vault: Path, paths: dict) -> int:` +- `migrate_to_workspace` is called inside `run_index_refresh()` before `build_index()` +- `_build_entry` in `asset_index.py` writes to workspace path when workspace dir exists +- `_build_entry` creates workspace subdirs (including `ai/`) when writing to workspace +- When workspace dir does NOT exist, `_build_entry` falls back to flat path writing (backward compat) +</action> + +<acceptance_criteria> +- `paperforge/worker/sync.py` contains `def migrate_to_workspace(vault: Path, paths: dict) -> int:` +- `run_index_refresh()` calls `migrate_to_workspace()` before `build_index()` +- `paperforge/worker/asset_index.py` `_build_entry()` writes note to workspace path when workspace dir exists +- `_build_entry()` creates `ai/` dir when writing to workspace +- Flat path fallback preserved when workspace dir does NOT exist +</acceptance_criteria> + +<verify> + <automated>python -c "from paperforge.worker.sync import migrate_to_workspace; print('migrate_to_workspace importable')"</automated> +</verify> + +<done> +migrate_to_workspace() exists in sync.py, is wired into run_index_refresh(), and _build_entry writes to workspace paths when workspace dir exists. Flat path fallback preserved. +</done> +</task> + +<task type="auto" tdd="false"> +<name>Task 2: Write tests for migration function, _build_entry workspace write, and idempotency</name> +<files> + tests/test_migration.py +</files> + +<read_first> + paperforge/worker/sync.py (migrate_to_workspace from Task 1) + paperforge/worker/asset_index.py (updated _build_entry from Task 1) + paperforge/worker/_utils.py (pipeline_paths, slugify_filename) + tests/test_sync.py (existing test patterns for mocking, fixture setup) +</read_first> + +<action> +Create `tests/test_migration.py` with the following test cases covering all 15 decisions (D-11 through D-15): + +Use `pytest` and the existing test fixture patterns from `tests/test_sync.py` (mock vault structure with `Literature/<domain>/` dir, fake canonical index, etc.). Use `tmp_path` for vault root. Use `monkeypatch` where needed for path resolution. + +**Test 1 — `test_migrate_flat_note_to_workspace` (D-11, D-12):** +- Create a mock vault with a flat note at `Literature/骨科/KEY123-title.md` +- Create a mock canonical index with one entry referencing that flat note +- Call `migrate_to_workspace(vault, paths)` +- Assert: workspace dir `Literature/骨科/KEY123-title/` is created +- Assert: main note at `Literature/骨科/KEY123-title/KEY123-title.md` has same content as flat note +- Assert: flat note `Literature/骨科/KEY123-title.md` still exists (D-12: copy-not-move) + +**Test 2 — `test_migrate_extracts_deep_reading` (D-13):** +- Create a flat note that contains a `## 🔍 精读` section with substantive content +- Call migrate_to_workspace +- Assert: `deep-reading.md` exists in workspace dir +- Assert: `deep-reading.md` content starts with `## 🔍 精读` and contains the deep reading content +- Assert: main note in workspace also contains the `## 🔍 精读` section (it's a complete copy) + +**Test 3 — `test_migrate_creates_ai_dir`:** +- Call migrate_to_workspace on a migrated paper +- Assert: `ai/` directory exists in workspace dir +- Assert: `ai/` directory is empty + +**Test 4 — `test_migrate_idempotent_skips_existing` (D-15):** +- Create a paper whose workspace dir already exists (previously migrated) +- Call migrate_to_workspace +- Assert: return count is 0 (nothing new migrated) +- Assert: workspace content is not overwritten (use a content marker to verify) + +**Test 5 — `test_build_entry_writes_to_workspace_after_migration`:** +- Create a mock vault with workspace dir already existing +- Call `_build_entry()` with a mock export item +- Assert: the note is written to `main_note_path` (workspace), not the flat path +- Assert: the written note has correct frontmatter + +**Test 6 — `test_build_entry_flat_fallback_for_unmigrated_paper`:** +- Create a mock vault with NO workspace dir but with a flat note +- Call `_build_entry()` with a mock export item +- Assert: the note is written to the flat path (backward compat) +- Assert: workspace dir is NOT created + +**Test 7 — `test_build_entry_new_paper_creates_workspace`:** +- Create a mock vault with neither flat note nor workspace dir (brand new paper) +- Call `_build_entry()` with a mock export item +- Assert: workspace dir is created +- Assert: note is written to workspace `main_note_path` +- Assert: `ai/` dir is created in workspace + +**Test 8 — `test_run_index_refresh_calls_migrate`:** +- Monkey-patch `migrate_to_workspace` with a spy +- Call `run_index_refresh()` +- Assert: `migrate_to_workspace` was called at least once + +Use `pytest.mark.parametrize` where appropriate. All tests must be fast (< 1s each) — no network or filesystem-side-effect dependencies. +</action> + +<acceptance_criteria> +- `tests/test_migration.py` exists with 8 test cases +- All 8 tests pass: `pytest tests/test_migration.py -x -q --tb=short` +- Each test covers a specific D-11 through D-15 scenario +- Tests use `tmp_path` and monkeypatch, no external state +</acceptance_criteria> + +<verify> + <automated>pytest tests/test_migration.py -x -q --tb=short</automated> +</verify> + +<done> +All 8 migration tests pass. Flat-to-workspace migration, _build_entry workspace writing, idempotency, and backward compatibility are verified. +</done> +</task> + +</tasks> + +<verification> +1. `pytest tests/test_migration.py -x -q --tb=short` — all 8 tests pass +2. `python -c "from paperforge.worker.sync import migrate_to_workspace; print('OK')"` — importable +3. Manual scan: `Literature/<domain>/` should now contain both flat notes AND workspace dirs after sync +4. Existing tests pass: `pytest tests/test_sync.py -x -q --tb=short` (no regressions) +</verification> + +<success_criteria> +- D-11: Existing flat notes migrated to workspace on first sync +- D-12: Original flat note preserved (copy-not-move) +- D-13: `## 🔍 精读` section extracted to `deep-reading.md` +- D-14: Workspace paths in canonical index point to real files +- D-15: Migration is idempotent, already-migrated papers skipped +- ai/ directory created in each paper workspace +- _build_entry writes to workspace path when workspace exists, falls back to flat for unmigrated +</success_criteria> + +<output> +After completion, create `.planning/phases/26-traceable-ai-context-packs/26-01-SUMMARY.md` +</output> diff --git a/.planning/phases/26-traceable-ai-context-packs/26-01-SUMMARY.md b/.planning/phases/26-traceable-ai-context-packs/26-01-SUMMARY.md new file mode 100644 index 00000000..aa0c31b5 --- /dev/null +++ b/.planning/phases/26-traceable-ai-context-packs/26-01-SUMMARY.md @@ -0,0 +1,110 @@ +--- +phase: 26-traceable-ai-context-packs +plan: 01 +subsystem: worker +tags: [migration, workspace, flat-to-workspace, canonical-index, idempotent] + +requires: + - phase: 22-configuration-truth-compatibility + provides: workspace path fields in canonical index (paper_root, main_note_path, etc.) + - phase: 23-canonical-asset-index-safe-rebuilds + provides: _build_entry(), build_index(), atomic writes, read_index() +provides: + - "migrate_to_workspace() function — copies flat notes to workspace dirs on first sync" + - "Workspace-aware _build_entry() — writes notes to workspace path when workspace dir exists" + - "Flat path fallback — backward compat for unmigrated papers" + - "Migration tests covering all D-11 through D-15 scenarios" +affects: + - "26-02: AI context commands will read workspace-structured assets" + - "26-03: Plugin context buttons will reference workspace paths" + +tech-stack: + added: [] + patterns: + - "Workspace path computation matches entry's main_note_path formula" + - "Flat path fallback preserves backward compatibility during transition" + - "idempotent migration: workspace_dir.exists() check before every copy" + +key-files: + created: + - tests/test_migration.py (9 tests for migration, _build_entry, and run_index_refresh) + modified: + - paperforge/worker/sync.py (migrate_to_workspace() + wired into run_index_refresh) + - paperforge/worker/asset_index.py (_build_entry workspace-aware write logic) + +key-decisions: + - "Migration runs before build_index() in run_index_refresh() so _build_entry sees the workspace dir" + - "When workspace dir exists, _build_entry writes to workspace path AND ensures ai/ dir exists" + - "When workspace dir does NOT exist, _build_entry falls back to flat path (no auto-create of workspace)" + - "New papers (first sync, no flat note) write to flat path; workspace creation deferred to migration pass" + +patterns-established: + - "Workspace directories are created by migrate_to_workspace(), not by _build_entry()" + - "The flat-to-workspace transition is a one-time migration pass, not per-entry logic" + - "Every workspace has ai/ subdir created during migration" + +requirements-completed: + - AIC-02 + - AIC-03 + - AIC-04 + +duration: 4 min +completed: 2026-05-04 +--- + +# Phase 26 Plan 01: Flat-to-Workspace Note Migration Summary + +**Flat literature notes copied into per-paper workspace directories with ## 🔍 精读 extraction and ai/ directory creation, wired into run_index_refresh() for first-sync migration, with workspace-aware _build_entry() writing and backward-compatible flat fallback** + +## Performance + +- **Duration:** 4 min +- **Started:** 2026-05-04T04:43:04Z +- **Completed:** 2026-05-04T04:46:36Z +- **Tasks:** 2 +- **Files modified:** 3 + +## Accomplishments + +- `migrate_to_workspace()` function in sync.py copies flat notes to workspace dirs, extracts deep-reading sections, and creates ai/ directories +- Migration wired into `run_index_refresh()` before `build_index()` — runs on every sync but skips already-migrated papers (idempotent per D-15) +- `_build_entry()` in asset_index.py updated to write notes to workspace path when workspace dir exists, with flat path fallback for backward compatibility +- 9 pytest tests covering migration, _build_entry workspace writing, idempotency, backward compatibility, and run_index_refresh integration + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Add migrate_to_workspace() and update _build_entry()** - `8e8677d` (feat) +2. **Task 2: Write migration tests** - `c753a96` (test) + +**Plan metadata:** *(will be committed in final metadata commit)* + +## Files Created/Modified + +| File | Action | Description | +|------|--------|-------------| +| `paperforge/worker/sync.py` | Modified | Added `migrate_to_workspace()` (~90 lines), wired into `run_index_refresh()` | +| `paperforge/worker/asset_index.py` | Modified | Updated `_build_entry()` note writing to support workspace paths + flat fallback | +| `tests/test_migration.py` | Created | 9 test cases covering all D-11 through D-15 scenarios | + +## Decisions Made + +- **Migration runs before build_index()**: This ensures `_build_entry()` sees the workspace dir and writes updates there, keeping existing deep-reading content in the workspace note. +- **Flat fallback is unconditional**: `_build_entry()` does NOT auto-create workspace dirs — it only reads/writes to workspace when the workspace dir already exists. This keeps the migration pass as the single source of truth for workspace creation. +- **Deep-reading preserved in both files**: The main workspace note is a full copy of the flat note (including `## 🔍 精读`), and the extracted deep-reading.md is a separate file. This provides redundancy — if deep-reading.md is lost, the content is still in the main note. + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered + +None. + +## Next Phase Readiness + +- Flat-to-workspace migration is complete — workspace directories now point to real files +- Ready for Plan 26-02 (AI context commands) which reads workspace-structured assets +- Ready for Plan 26-03 (Plugin context buttons) referencing workspace paths +- Existing tests continue to pass (44 passed in asset index and asset state) diff --git a/.planning/phases/26-traceable-ai-context-packs/26-02-PLAN.md b/.planning/phases/26-traceable-ai-context-packs/26-02-PLAN.md new file mode 100644 index 00000000..200fbeb0 --- /dev/null +++ b/.planning/phases/26-traceable-ai-context-packs/26-02-PLAN.md @@ -0,0 +1,477 @@ +--- +phase: 26-traceable-ai-context-packs +plan: 02 +type: execute +wave: 1 +depends_on: [] +files_modified: + - paperforge/commands/context.py + - paperforge/commands/__init__.py + - paperforge/cli.py + - tests/test_context.py +autonomous: true +requirements: + - AIC-02 + - AIC-03 + - AIC-04 + +must_haves: + truths: + - "paperforge context <key> prints a single canonical index entry as JSON" + - "paperforge context --domain <domain> prints all entries in that domain as a JSON array" + - "paperforge context --collection <path> prints entries matching that collection path as a JSON array" + - "paperforge context --all prints all entries as a JSON array" + - "Each entry in the output clearly lists its included metadata, fulltext, note links, and provenance paths" + - "When a paper cannot produce AI context, the output explains what source assets are missing or why generation is blocked" + - "Every item in a context pack can be traced back to its originating PDF, OCR output, and formal note" + artifacts: + - path: paperforge/commands/context.py + provides: "CLI context command — reads canonical index, formats JSON output with provenance" + - path: paperforge/cli.py + provides: "context subparser + main dispatch" + - path: tests/test_context.py + provides: "Tests for context command output, filtering, and blocking explanations" + key_links: + - from: paperforge/commands/context.py:run + to: paperforge/worker/asset_index:read_index + via: "read canonical index, filter by key/domain/collection/all, format output" + pattern: "from paperforge\\.worker\\.asset_index import.*read_index" + - from: paperforge/commands/context.py output entry + to: paperforge/worker/asset_index:_build_entry workspace paths + via: "output references paper_root, main_note_path, fulltext_path, ocr_md_path, pdf_path from index entry" + pattern: "paper_root|main_note_path|fulltext_path" +--- + +<objective> +Provide a `paperforge context` CLI command that outputs canonical index entries as JSON, forming the traceable AI entry point. Per D-01, the canonical index entry IS the AI context — no separate pack format is designed. The command reads from the existing canonical index, formats entries with provenance, and explains blocking when AI context is not ready. + +Purpose: AIC-02 (single paper context), AIC-03 (collection-level context), AIC-04 (explainable entry points). The CLI command is the core AI integration surface: Agent skills or scripts can call `paperforge context <key>` to get a self-describing JSON bundle that includes all source asset links and readiness explanations. + +Output: +- `paperforge/commands/context.py` — context command with single-key, --domain, --collection, --all modes +- `paperforge/cli.py` — context subparser + main dispatch +- `tests/test_context.py` — tests for all command modes and provenance output +</objective> + +<execution_context> +@/C/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@/C/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> +@.planning/STATE.md +@.planning/ROADMAP.md +@.planning/phases/26-traceable-ai-context-packs/26-CONTEXT.md + +<interfaces> +Command module pattern (see paperforge/commands/status.py): +```python +def run(args: argparse.Namespace) -> int: + vault = getattr(args, "vault_path", None) + if vault is None: + from paperforge.config import resolve_vault + vault = resolve_vault(cli_vault=getattr(args, "vault", None)) + # ... implementation ... + return exit_code +``` + +Command registry (paperforge/commands/__init__.py): +```python +_COMMAND_REGISTRY: dict[str, str] = { + "sync": "paperforge.commands.sync", + # ... add "context": "paperforge.commands.context", +} +``` + +CLI dispatch pattern (paperforge/cli.py main function, lines 358-408): +```python +if args.command == "sync": + from paperforge.commands import sync + return sync.run(args) +``` + +Canonical index entry shape (from asset_index.py _build_entry): +```python +{ + "zotero_key": str, # "ABCDEFG" + "domain": str, # "骨科" + "title": str, # Full title + "authors": list[str], # ["Author A", "Author B"] + "abstract": str, # Paper abstract text + "journal": str, # Journal name + "year": str, # "2024" + "doi": str, # "10.xxxx/xxxxx" + "pmid": str, # PubMed ID + "collection_path": str, # "Parent | Child" + "collections": list[str], # ["骨科", "运动医学"] + "collection_tags": list[str],# ["review"] + "has_pdf": bool, + "pdf_path": str, # "[[wikilink to PDF]]" + "ocr_status": str, # "pending" | "processing" | "done" | "failed" + "ocr_md_path": str, # "[[wikilink to fulltext.md]]" + "ocr_json_path": str, # path to OCR meta.json + "deep_reading_status": str, # "pending" | "done" + "note_path": str, # flat path (legacy) + "deep_reading_md_path": str, # workspace deep-reading.md path + # Workspace paths (canonical): + "paper_root": str, # "Literature/骨科/KEY - Title/" + "main_note_path": str, # "Literature/骨科/KEY - Title/KEY - Title.md" + "fulltext_path": str, # "Literature/骨科/KEY - Title/fulltext.md" + "deep_reading_path": str, # "Literature/骨科/KEY - Title/deep-reading.md" + "ai_path": str, # "Literature/骨科/KEY - Title/ai/" + # Derived state: + "lifecycle": str, # "indexed" | "pdf_ready" | "fulltext_ready" | "deep_read_done" | "ai_context_ready" + "health": dict, # {pdf_health, ocr_health, note_health, asset_health} + "maturity": dict, # {level, level_name, checks, blocking} + "next_step": str, # "sync" | "ocr" | "repair" | "/pf-deep" | "rebuild index" | "ready" +} +``` + +Canonical index envelope (from asset_index.py build_envelope): +```python +{ + "schema_version": "2", + "generated_at": "2026-05-04T12:34:56Z", + "paper_count": int, + "items": [entry, ...] +} +``` + +CLI parser build pattern (from cli.py build_parser): +```python +p_context = sub.add_parser("context", help="Generate traceable AI context pack for paper(s)") +p_context.add_argument("key", nargs="?", help="Zotero citation key for single paper context") +p_context.add_argument("--domain", metavar="DOMAIN", help="Filter by domain") +p_context.add_argument("--collection", metavar="PATH", help="Filter by collection path (prefix match)") +p_context.add_argument("--all", action="store_true", help="All papers in the canonical index") +``` + +Decisions from CONTEXT.md: +- D-01: Canonical index entry IS the AI context. No separate "context pack" format. +- D-02: AI reads the index entry JSON + referenced assets directly. +- D-05: `paperforge context <key>` for single paper, `--domain`, `--collection`, `--all` for multi-entry. +- D-06: Output is JSON. Default to individual entry (dict), array for multi-entry modes. +- D-09: Provenance is inherent: entries already have `paper_root`, `main_note_path`, `fulltext_path`, `ocr_path`, `note_path`. No additional layer needed. +- D-10: No new dependencies. Everything uses existing asset_index.py index reading. +</interfaces> +</context> + +<tasks> + +<task type="auto" tdd="false"> +<name>Task 1: Create context command module with single-key output and provenance</name> +<files> + paperforge/commands/context.py +</files> + +<read_first> + paperforge/commands/status.py (command module pattern — run(args) function signature) + paperforge/worker/asset_index.py (read_index, get_index_path, build_envelope) +</read_first> + +<action> +Create `paperforge/commands/context.py` implementing the core context command. + +The module must have a `run(args: argparse.Namespace) -> int` entry point that: + +1. **Reads the canonical index** via `read_index(vault)` from `paperforge.worker.asset_index`. + +2. **Filters entries based on the mode:** + - If `args.key` is provided: find the single entry whose `zotero_key` matches. Case-sensitive, exact match. + - If `args.domain` is provided: filter entries where `entry["domain"]` equals the argument (exact match per the agent's discretion). + - If `args.collection` is provided: filter entries where any element in `entry["collections"]` starts with the argument value (prefix match per the agent's discretion). Use `startswith` for prefix matching. + - If `args.all` is true: include all entries. + - If none specified and no key given: print error and return 1. + +3. **Formats each entry for output.** Per D-09, the entry already carries all necessary provenance paths. The output format wraps each entry with a `_provenance` block and an `_ai_readiness` block: + + ```python + def _format_context_entry(entry: dict) -> dict: + """Wrap a canonical index entry with provenance and readiness metadata.""" + # Determine AI readiness + lifecycle = entry.get("lifecycle", "indexed") + health = entry.get("health", {}) + blocking = entry.get("maturity", {}).get("blocking", []) + + readiness = { + "ai_context_ready": lifecycle == "ai_context_ready", + "lifecycle": lifecycle, + "maturity_level": entry.get("maturity", {}).get("level", 1), + "blocking_factors": blocking, + "next_step": entry.get("next_step", ""), + } + + # If not ai_context_ready, explain why (AIC-04) + if lifecycle != "ai_context_ready": + reasons = [] + if not entry.get("has_pdf"): + reasons.append("No PDF attachment — run sync to import") + elif entry.get("ocr_status") != "done": + reasons.append(f"OCR status is '{entry.get('ocr_status', 'pending')}' — run ocr") + elif entry.get("deep_reading_status") != "done": + reasons.append("Deep reading not complete — run /pf-deep") + if health and any(v != "healthy" for v in health.values()): + unhealthy = [k for k, v in health.items() if v != "healthy"] + reasons.append(f"Unhealthy asset(s): {', '.join(unhealthy)} — run repair") + readiness["blocking_explanation"] = "; ".join(reasons) if reasons else "Unknown blocking state" + + # Build provenance trace (AIC-02, AIC-04) + provenance = { + "paper_root": entry.get("paper_root", ""), + "main_note_path": entry.get("main_note_path", ""), + "fulltext_path": entry.get("fulltext_path", ""), + "ocr_md_path": entry.get("ocr_md_path", ""), + "pdf_path": entry.get("pdf_path", ""), + "deep_reading_path": entry.get("deep_reading_path", ""), + "ai_path": entry.get("ai_path", ""), + "note_path": entry.get("note_path", ""), + "deep_reading_md_path": entry.get("deep_reading_md_path", ""), + } + + # Assemble output: metadata subset + provenance + readiness + output = { + "_format_version": "1", + "_context": "Canonical index entry — the AI context per PaperForge D-01", + "_generated_at": datetime.now(timezone.utc).isoformat(), + "zotero_key": entry["zotero_key"], + "domain": entry.get("domain", ""), + "title": entry.get("title", ""), + "authors": entry.get("authors", []), + "abstract": entry.get("abstract", ""), + "journal": entry.get("journal", ""), + "year": entry.get("year", ""), + "doi": entry.get("doi", ""), + "pmid": entry.get("pmid", ""), + "collections": entry.get("collections", []), + "_provenance": provenance, + "_ai_readiness": readiness, + } + return output + ``` + +4. **Outputs JSON:** + - Single entry (key mode): print the formatted dict as JSON. + - Multi-entry (--domain, --collection, --all): print a JSON array of formatted dicts. + - Use `json.dumps(output, ensure_ascii=False, indent=2)` for human-readable output. + - If no entries match the filter: print `[]` for multi-entry modes, print error and return 1 for key mode. + +5. **If key is specified but not found:** print error message and return 1. + +Use lazy imports (`from paperforge.worker.asset_index import read_index` inside `run()`) to follow the established pattern from other command modules. + +Import datetime inside the function: +```python +from datetime import datetime, timezone +``` + +The module should be importable without triggering CLI argument parsing. +</action> + +<acceptance_criteria> +- `paperforge/commands/context.py` exists with `def run(args: argparse.Namespace) -> int:` +- `run()` reads canonical index and formats entries with `_provenance` and `_ai_readiness` blocks +- Single key mode outputs a single JSON object, multi-entry mode outputs a JSON array +- Blocking explanation included when lifecycle != "ai_context_ready" +- Missing key returns exit code 1 with error message +</acceptance_criteria> + +<verify> + <automated>python -c "from paperforge.commands.context import run; print('context module importable')"</automated> +</verify> + +<done> +context command module exists with single-key context output, provenance trace, and AI readiness explanation. +</done> +</task> + +<task type="auto" tdd="false"> +<name>Task 2: Wire context command into CLI (subparser + main dispatch + command registry)</name> +<files> + paperforge/cli.py + paperforge/commands/__init__.py +</files> + +<read_first> + paperforge/cli.py (build_parser function lines 118-289, main function lines 319-451) + paperforge/commands/__init__.py (command registry) +</read_first> + +<action> +**Change A — Register in command registry (`paperforge/commands/__init__.py`):** +Add to `_COMMAND_REGISTRY`: +```python +"context": "paperforge.commands.context", +``` + +**Change B — Add subparser in `build_parser()` (`paperforge/cli.py`):** +After the `ocr` subparser block (around line 212-220) or before the `base-refresh` parser, add: +```python +# context (Phase 26: traceable AI context packs) +p_context = sub.add_parser("context", help="Generate traceable AI context pack for paper(s)") +p_context.add_argument( + "key", + nargs="?", + metavar="KEY", + help="Zotero citation key for a single paper (outputs single JSON object)", +) +p_context.add_argument( + "--domain", + metavar="DOMAIN", + help="Filter by domain (outputs JSON array)", +) +p_context.add_argument( + "--collection", + metavar="PATH", + help="Filter by collection path (prefix match, outputs JSON array)", +) +p_context.add_argument( + "--all", + action="store_true", + help="Output all entries in the canonical index (JSON array)", +) +``` + +**Change C — Add dispatch in `main()` (`paperforge/cli.py`):** +After the `deep-reading` dispatch block (around line 402) or before `base-refresh`, add: +```python +if args.command == "context": + from paperforge.commands import context + return context.run(args) +``` + +Do NOT add `--json` flag — the context command always outputs JSON (per D-06). + +Validation on dispatch: at least one of `key`, `--domain`, `--collection`, `--all` must be provided. The context module's `run()` already handles this validation; no need to duplicate in CLI. + +Make sure `argparse` properly handles `nargs="?"` for the optional key argument. +</action> + +<acceptance_criteria> +- `paperforge/commands/__init__.py` registers `"context": "paperforge.commands.context"` +- `paperforge/cli.py` `build_parser()` creates `context` subparser with `key`, `--domain`, `--collection`, `--all` arguments +- `paperforge/cli.py` `main()` dispatches `context` to `paperforge.commands.context.run(args)` +- `python -m paperforge context --help` shows usage with all arguments +</acceptance_criteria> + +<verify> + <automated>python -m paperforge context --help</automated> +</verify> + +<done> +CLI wiring complete: `paperforge context` command available with all four access modes (key, --domain, --collection, --all). +</done> +</task> + +<task type="auto" tdd="false"> +<name>Task 3: Write tests for context command modes, filtering, and provenance output</name> +<files> + tests/test_context.py +</files> + +<read_first> + paperforge/commands/context.py (run function from Task 1) + paperforge/worker/asset_index.py (read_index, build_envelope) + tests/test_status.py (existing test patterns for CLI command testing) +</read_first> + +<action> +Create `tests/test_context.py` with comprehensive tests. Use `tmp_path` for vault root and `monkeypatch` for path resolution. Create mock canonical index files using `atomic_write_index` or `json.dump` directly. + +**Test fixture:** Create a helper function `_make_index(vault, entries)` that writes a valid canonical index envelope with the given entries to `get_index_path(vault)`. + +**Test 1 — `test_context_single_key` (AIC-02):** +- Create a mock index with 2 entries (keys: "KEY001", "KEY002") +- Build args Namespace with `key="KEY001"`, `domain=None`, `collection=None`, `all=False` +- Call `run(args)` +- Assert: prints a single JSON object (not array) +- Assert: printed JSON has `zotero_key: "KEY001"` +- Assert: printed JSON has `_provenance` with all path fields +- Assert: printed JSON has `_ai_readiness` with `ai_context_ready` and `lifecycle` + +**Test 2 — `test_context_single_key_not_found`:** +- Create mock index with "KEY001" only +- Build args with `key="KEY999"` +- Call `run(args)` +- Assert: returns 1 (error) +- Assert: prints error message containing "not found" or similar + +**Test 3 — `test_context_domain_filter` (AIC-03):** +- Create mock index with entries in "骨科", "运动医学", "骨科" domains +- Build args with `domain="骨科"`, `key=None`, `collection=None`, `all=False` +- Call `run(args)` +- Assert: prints a JSON array +- Assert: array has 2 entries, both with `domain: "骨科"` + +**Test 4 — `test_context_collection_filter` (AIC-03):** +- Create mock index with entries having `collections: ["骨科/脊柱", "骨科/关节"]`, `collections: ["运动医学"]`, `collections: ["骨科/创伤"]` +- Build args with `collection="骨科"`, `key=None`, `domain=None`, `all=False` +- Call `run(args)` +- Assert: prints JSON array with 2 entries (both starting with "骨科") +- Assert: the "运动医学" entry is excluded + +**Test 5 — `test_context_all`:** +- Create mock index with 3 entries +- Build args with `all=True`, `key=None`, `domain=None`, `collection=None` +- Call `run(args)` +- Assert: prints JSON array with all 3 entries + +**Test 6 — `test_context_provenance_traceability` (AIC-04):** +- Create a mock entry with all fields populated +- Build args with `key` matching that entry +- Assert: output `_provenance` contains all path keys: `paper_root`, `main_note_path`, `fulltext_path`, `ocr_md_path`, `pdf_path`, `deep_reading_path`, `ai_path`, `note_path` +- Assert: all path values are non-empty strings + +**Test 7 — `test_context_ai_readiness_blocking_explanation` (AIC-04):** +- Create entries at different lifecycle stages: "pdf_ready" (no fulltext), "fulltext_ready" (no deep read), "ai_context_ready" +- For the "pdf_ready" entry: assert `_ai_readiness.blocking_explanation` contains information about missing OCR or deep reading +- For the "ai_context_ready" entry: assert `_ai_readiness.ai_context_ready` is true, `blocking_factors` is empty + +**Test 8 — `test_context_no_entries_match`:** +- Create mock index with "KEY001" only +- Build args with `domain="Nonexistent"` +- Call `run(args)` +- Assert: prints `[]` +- Assert: returns 0 (empty array is success, no error) + +Use `pytest` with `unittest.mock.patch` or `monkeypatch` to control the vault path. Use `argparse.Namespace` for args construction: +```python +import argparse +args = argparse.Namespace(key="KEY001", domain=None, collection=None, all=False, vault_path=tmp_path) +``` +</action> + +<acceptance_criteria> +- `tests/test_context.py` exists with 8+ test cases covering all modes +- All tests pass: `pytest tests/test_context.py -x -q --tb=short` +- Tests cover single key, domain filter, collection filter, all, missing key, no-match, provenance content, and blocking explanation +</acceptance_criteria> + +<verify> + <automated>pytest tests/test_context.py -x -q --tb=short</automated> +</verify> + +<done> +All 8+ context command tests pass. Context command is verified for single-key, domain, collection, all modes, provenance, and blocking explanations. +</done> +</task> + +</tasks> + +<verification> +1. `python -m paperforge context --help` shows usage with all arguments +2. `pytest tests/test_context.py -x -q --tb=short` — all tests pass +3. `python -c "from paperforge.commands.context import run; from paperforge.commands import get_command_module; m = get_command_module('context'); print('registry OK')"` — registry works +4. Manual test: create a mock canonical index and run `python -m paperforge context <key>` on it +</verification> + +<success_criteria> +- AIC-02: `paperforge context <key>` outputs traceable context with metadata, fulltext, note links, and provenance +- AIC-03: `paperforge context --domain` and `--collection` outputs filtered context without hardcoded extraction schemas +- AIC-04: Each entry includes blocking explanation when AI context is not ready, and provenance paths for every source asset +- D-09: Provenance is inherent — no additional layer needed, entry paths are sufficient +- D-10: No new dependencies beyond existing asset_index.py +</success_criteria> + +<output> +After completion, create `.planning/phases/26-traceable-ai-context-packs/26-02-SUMMARY.md` +</output> diff --git a/.planning/phases/26-traceable-ai-context-packs/26-03-PLAN.md b/.planning/phases/26-traceable-ai-context-packs/26-03-PLAN.md new file mode 100644 index 00000000..b8a9b4fa --- /dev/null +++ b/.planning/phases/26-traceable-ai-context-packs/26-03-PLAN.md @@ -0,0 +1,462 @@ +--- +phase: 26-traceable-ai-context-packs +plan: 03 +type: execute +wave: 2 +depends_on: + - "26-02" +files_modified: + - paperforge/plugin/main.js +autonomous: true +requirements: + - AIC-02 + - AIC-03 + - AIC-04 + +must_haves: + truths: + - "User can click a 'Copy Context' button from any paper note to copy that paper's canonical index entry JSON to clipboard" + - "User can click a 'Copy Collection Context' button from the PaperForge dashboard to copy filtered context JSON to clipboard" + - "Copied JSON is identical to what paperforge context <key> would output" + - "If a paper cannot provide AI context, the copied output explains why" + - "The 'Copy Context' action is available from the command palette" + artifacts: + - path: paperforge/plugin/main.js + provides: "Two new Quick Actions: 'Copy Context' (per-paper) and 'Copy Collection Context' (multi-paper)" + key_links: + - from: paperforge/plugin/main.js ACTIONS entry + to: paperforge CLI context command + via: "spawn('python', ['-m', 'paperforge', 'context', key], ...) in _runAction" + pattern: "context|Copy Context" +--- + +<objective> +Add "Copy Context" and "Copy Collection Context" buttons to the Obsidian plugin, providing GUI access to the `paperforge context` CLI command. These are the user-facing entry points for AI context packs (AIC-02, AIC-03, AIC-04). + +Purpose: Users interact with PaperForge primarily through the Obsidian plugin. CLI-only context generation would force users to switch to terminal. Plugin buttons make context generation a one-click operation from the dashboard or from any paper note. + +Output: +- `paperforge/plugin/main.js` — Two new ACTIONS entries, key-resolution logic, clipboard output +</objective> + +<execution_context> +@/C/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@/C/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> +@.planning/STATE.md +@.planning/ROADMAP.md +@.planning/phases/26-traceable-ai-context-packs/26-CONTEXT.md + +<interfaces> +Plugin ACTIONS array pattern (paperforge/plugin/main.js lines 157-190): +```javascript +const ACTIONS = [ + { + id: 'paperforge-sync', + title: 'Sync Library', + desc: 'Pull new references from Zotero and generate literature notes', + icon: '\u21BB', // ↻ + cmd: 'sync', + okMsg: 'Sync complete', + }, + // ...more actions... +]; +``` + +Plugin action execution (main.js _runAction, lines 446-493): +```javascript +_runAction(a, card) { + card.addClass('running'); + const vp = this.app.vault.adapter.basePath; + this._showMessage('Processing...', 'running'); + const { spawn } = require('node:child_process'); + const child = spawn('python', ['-m', 'paperforge', a.cmd], { cwd: vp, timeout: 600000 }); + // stdout/stderr handlers, close handler with clipboard copy + child.on('close', (code) => { + if (code === 0) { + // Copy output to clipboard + const output = log.join('\n'); + navigator.clipboard.writeText(output); + new Notice('[OK] Context copied to clipboard'); + } + }); +} +``` + +The current `_runAction` uses `a.cmd` directly as the subcommand. For context actions, the command needs additional arguments (zotero key or filter flags). + +CLI context command (from Plan 26-02): +- `python -m paperforge context <key>` — single paper +- `python -m paperforge context --domain <domain>` — domain filter +- `python -m paperforge context --collection <path>` — collection filter +- `python -m paperforge context --all` — all entries + +Decisions from CONTEXT.md: +- D-07: Plugin adds "Copy Context" button for the current paper (copies canonical index entry JSON). +- D-08: Plugin adds "Copy Collection Context" button (from Base view or dashboard, copies filtered entries). +</interfaces> +</context> + +<tasks> + +<task type="auto" tdd="false"> +<name>Task 1: Add "Copy Context" Quick Action with active-paper key resolution</name> +<files> + paperforge/plugin/main.js +</files> + +<read_first> + paperforge/plugin/main.js:157-190 (ACTIONS array definition) + paperforge/plugin/main.js:446-493 (_runAction implementation) +</read_first> + +<action> +Add a "Copy Context" action to the ACTIONS array and update `_runAction` to support key-specific commands. + +**Change A — Extend ACTIONS array:** +Add a new entry to ACTIONS: +```javascript +{ + id: 'paperforge-copy-context', + title: 'Copy Context', + desc: 'Copy this paper\u2019s canonical index entry JSON to clipboard for AI use', + icon: '\u2139', // ℹ + cmd: 'context', // CLI subcommand + needsKey: true, // NEW: signals this action needs the current paper's zotero_key + okMsg: 'Context copied', +}, +``` + +The `needsKey: true` flag is new. Other actions (sync, ocr, doctor, repair) set `needsKey: false` or omit it. + +**Change B — Update `_runAction` to handle key-specific commands and clipboard copy:** + +Modify `_runAction(a, card)` to: + +1. Resolve the zotero key when `a.needsKey` is true: + ```javascript + let extraArgs = []; + if (a.needsKey) { + // Try to get the active file's zotero_key from its frontmatter + const activeFile = this.app.workspace.getActiveFile(); + let key = null; + if (activeFile) { + const cache = this.app.metadataCache.getFileCache(activeFile); + if (cache && cache.frontmatter && cache.frontmatter.zotero_key) { + key = cache.frontmatter.zotero_key; + } + } + if (!key) { + this._showMessage('[!!] No zotero_key found in active note frontmatter', 'error'); + new Notice('[!!] Open a paper note with a zotero_key in its frontmatter first', 6000); + card.removeClass('running'); + return; + } + extraArgs = [key]; + } + ``` + +2. Build the spawn command with extra args: + ```javascript + const child = spawn('python', ['-m', 'paperforge', a.cmd, ...extraArgs], { cwd: vp, timeout: 30000 }); + ``` + +3. On successful completion (close code === 0), copy output to clipboard: + ```javascript + child.on('close', (code) => { + clearInterval(pollTimer); + card.removeClass('running'); + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + if (code !== 0) { + const last = log.slice(-3).join(' | ') || 'exit code ' + code; + this._showMessage('[!!] ' + last, 'error'); + new Notice('[!!] ' + a.cmd + ' failed: ' + last, 8000); + } else { + // Copy all logged output to clipboard + const output = log.join('\n'); + if (output.trim()) { + try { + // Validate JSON before copying + JSON.parse(output); + navigator.clipboard.writeText(output).then(() => { + const summary = `${elapsed}s \u2014 ${output.length} chars copied`; + this._showMessage('[OK] ' + a.title + ': ' + summary, 'ok'); + new Notice('[OK] ' + a.okMsg + ' \u2014 ' + output.length + ' chars'); + }).catch((err) => { + this._showMessage('[!!] Clipboard write failed: ' + err.message, 'error'); + new Notice('[!!] Clipboard error', 6000); + }); + } catch (e) { + this._showMessage('[!!] Invalid JSON from context command', 'error'); + new Notice('[!!] Context command returned invalid JSON', 8000); + } + } else { + this._showMessage('[!!] No output from context command', 'error'); + new Notice('[!!] Context command returned empty output', 8000); + } + this._fetchStats(true); + } + }); + ``` + +4. For existing actions (sync, ocr, doctor, repair) that don't set `needsKey`, keep existing behavior: `extraArgs` stays empty, timeout stays at 600000. + +**Change C — Add command palette registration for "Copy Context":** + +After the existing commands block (around line 1086), register a command palette entry: +```javascript +this.addCommand({ + id: 'paperforge-copy-context', + name: 'Copy Context: Copy active paper\u2019s index entry JSON to clipboard', + callback: () => { + // Find the copy-context action and trigger it + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) { + // Create a temporary card element for the loading UI + const tempCard = createEl('div'); + this._runAction(action, tempCard); + } + }, +}); +``` + +This command palette entry is registered inside `onload()` (the plugin's main initialization function). Find the `onload()` method and add the `this.addCommand(...)` call there. + +Keep all existing ACTIONS entries unchanged. +</action> + +<acceptance_criteria> +- ACTIONS array has new entry with `id: 'paperforge-copy-context'`, `needsKey: true`, `cmd: 'context'` +- `_runAction()` checks `a.needsKey` and resolves zotero_key from active file frontmatter +- When zotero_key is missing, shows error notice and returns early +- On successful context output, copies JSON to clipboard via `navigator.clipboard.writeText()` +- Validates JSON before copying (prevents clipboard pollution) +- Command palette entry registered: `Copy Context: Copy active paper's index entry JSON to clipboard` +</acceptance_criteria> + +<verify> + <automated>MISSING — Wave 0 must create test scaffolding first. Verify manually: open Obsidian, open a paper note with zotero_key frontmatter, click "Copy Context" action, paste clipboard content to verify valid JSON.</automated> +</verify> + +<done> +"Copy Context" button added to plugin dashboard. Reads zotero_key from active note, calls `paperforge context <key>`, copies JSON output to clipboard. Command palette entry also available. +</done> +</task> + +<task type="auto" tdd="false"> +<name>Task 2: Add "Copy Collection Context" Quick Action for multi-paper context</name> +<files> + paperforge/plugin/main.js +</files> + +<read_first> + paperforge/plugin/main.js (ACTIONS array, _runAction, _fetchStats method) +</read_first> + +<action> +Add a "Copy Collection Context" action that copies context for all papers in the current view or filter. + +**Change A — Add to ACTIONS array:** +```javascript +{ + id: 'paperforge-copy-collection-context', + title: 'Copy Collection Context', + desc: 'Copy canonical index entries for all visible papers to clipboard', + icon: '\u2261', // ≡ + cmd: 'context', + needsFilter: true, // NEW: signals this action needs a filter flag + okMsg: 'Collection context copied', +}, +``` + +The `needsFilter: true` flag is new. It means this action should add a filter argument to the CLI command (e.g. `--all`, `--domain`, or `--collection`). + +**Change B — Update `_runAction` filter resolution:** + +Add filter resolution logic after the key resolution block: +```javascript +if (a.needsKey) { + // ... existing key resolution from Task 1 ... +} +if (a.needsFilter) { + // Default to --all if no specific filter can be determined + let filterArg = '--all'; + + // Phase 25 dashboard stores domain stats; try to read current domain filter + // For now, default to --all (the agent's discretion, per "the agent's Discretion" in CONTEXT.md) + // Future enhancement: read active Base view filter + + extraArgs = [filterArg]; +} +``` + +**Change C — Handle collection context output in close handler:** + +The close handler already handles clipboard copy from Task 1. The collection context output is a JSON array (since `--all` / `--domain` / `--collection` output is always an array per D-06). The existing clipboard logic in Task 1's close handler already handles this — no additional changes needed for the output handling. + +**Key design decisions:** +- Default to `--all` for the initial implementation (per "the agent's Discretion" in CONTEXT.md about plugin button placement) +- The output is a JSON array, which may be large. The clipboard `writeText` API handles up to ~1MB on most platforms. For very large libraries, users may need to use the CLI directly. +- The dashboard button triggers on click from any paper. The collection context represents all papers in the canonical index (or the current filtered view in a future enhancement). + +**Change D — Add command palette entry:** + +After the "Copy Context" command registration, add: +```javascript +this.addCommand({ + id: 'paperforge-copy-collection-context', + name: 'Copy Collection Context: Copy all canonical index entries to clipboard', + callback: () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-collection-context'); + if (action) { + const tempCard = createEl('div'); + this._runAction(action, tempCard); + } + }, +}); +``` + +Add both command palette entries inside the plugin's `onload()` method. Look for the existing command palette entries (there should be one like `paperforge: Open Dashboard` at around line 1125): +```javascript +// Find existing command registration block and add after it: +this.addCommand({ + id: 'paperforge-copy-context', + name: 'Copy Context: Copy active paper...', + callback: () => { ... } +}); +this.addCommand({ + id: 'paperforge-copy-collection-context', + name: 'Copy Collection Context: Copy all canonical...', + callback: () => { ... } +}); +``` +</action> + +<acceptance_criteria> +- ACTIONS array has entry with `id: 'paperforge-copy-collection-context'`, `needsFilter: true` +- `_runAction()` adds `--all` filter argument when `a.needsFilter` is true +- Dashboard shows the "Copy Collection Context" card in Quick Actions grid +- Command palette has entry for "Copy Collection Context" +</acceptance_criteria> + +<verify> + <automated>MISSING — Requires manual Obsidian testing. Open plugin dashboard, verify "Copy Collection Context" card appears in Quick Actions grid. Click it, paste clipboard content to see JSON array of all entries.</automated> +</verify> + +<done> +"Copy Collection Context" button added to plugin dashboard. Copies JSON array of all canonical index entries to clipboard. Command palette entry registered. +</done> +</task> + +<task type="auto" tdd="false"> +<name>Task 3: Error handling, edge cases, and integration robustness</name> +<files> + paperforge/plugin/main.js +</files> + +<read_first> + paperforge/plugin/main.js (full _runAction, the ACTIONS array, Dashboard onOpen) +</read_first> + +<action> +Add error handling and edge case coverage for the two new actions: + +**Change A — Zotero key not found in active note:** +Already implemented in Task 1 (shows error notice). Verify the error flow: +- When `needsKey` is true and `app.workspace.getActiveFile()` is null (no note open): show error +- When `needsKey` is true and frontmatter exists but has no `zotero_key`: show error mentioning the missing property +- When `needsKey` is true and there's no frontmatter at all: show error + +**Change B — Context command returns empty or invalid JSON:** +Already implemented in Task 1 (JSON.parse validation). Ensure the error message is specific: "Context command returned invalid JSON" rather than a generic error. + +**Change C — Large collection context timeout:** +The collection context (--all) may be large for vaults with many papers. Increase timeout for `needsFilter` actions: +```javascript +const cmdTimeout = a.needsFilter ? 60000 : (a.needsKey ? 30000 : 600000); +const child = spawn('python', ['-m', 'paperforge', a.cmd, ...extraArgs], { cwd: vp, timeout: cmdTimeout }); +``` + +**Change D — Loading state for new actions:** +Ensure the action card's "running" state works properly: +- The `card.addClass('running')` and `card.removeClass('running')` calls in `_runAction` work for dashboard cards +- For command palette invocations (which pass a temporary card element), removing the class is a no-op since the element is never added to DOM. This is acceptable — the notice provides feedback. + +**Change E — Multiple rapid clicks:** +Add a guard to prevent running the same action simultaneously: +```javascript +if (card.classList.contains('running')) { + return; // Already running +} +``` +Add this at the very beginning of `_runAction`, before any other logic. + +**Change F — Close handler JSON validation robustness:** +Replace the simple `JSON.parse(output)` with a try-catch: +```javascript +try { + JSON.parse(output); +} catch (e) { + this._showMessage('[!!] Invalid JSON from ' + a.title, 'error'); + new Notice('[!!] ' + a.title + ' returned invalid JSON: ' + e.message.slice(0, 100), 8000); + return; +} +``` + +Wrap these error-handling additions inside the existing `_runAction` method without disrupting the existing close handler logic for sync/ocr/doctor/repair actions. The existing `log` accumulation, polling, and elapsed-time display must remain unchanged for those actions. + +**Change G — Verify ACTIONS integration with command palette:** +All ACTIONS entries should still render in the dashboard. Verify the new actions appear in the Quick Actions grid by checking that the `_buildPanel()` method's rendering loop continues to work: +```javascript +for (const a of ACTIONS) { + const card = actionsGrid.createEl('div', { cls: 'paperforge-action-card' }); + // ... render logic ... +} +``` +Any new ACTIONS entries are automatically picked up by this loop. +</action> + +<acceptance_criteria> +- Multiple rapid clicks on same action card are blocked (running guard) +- Missing zotero_key shows specific error notice +- Missing active file shows specific error notice +- Large collection context uses higher timeout (60000ms) +- Invalid JSON from context command shows specific error message +- All existing actions (sync, ocr, doctor, repair) continue to work unchanged +- New actions appear correctly in the Quick Actions grid +</acceptance_criteria> + +<verify> + <automated>MISSING — Manual Obsidian verification needed. Test scenarios: (1) Click "Copy Context" with no active note — should show error. (2) Click "Copy Context" on a paper without zotero_key — should show error. (3) Click rapidly twice on "Copy Context" — second click should be blocked. (4) Click "Copy Collection Context" — should copy valid JSON array.</automated> +</verify> + +<done> +Error handling and edge cases covered for both new actions. Existing actions unaffected. Rapid-click guard prevents double-execution. +</done> +</task> + +</tasks> + +<verification> +1. Manual: Open Obsidian with plugin loaded. Verify dashboard shows all 6 Quick Actions (4 existing + 2 new). +2. Manual: Open a paper note with zotero_key in frontmatter. Click "Copy Context". Paste clipboard — should see valid JSON with provenance and readiness blocks. +3. Manual: Open Command Palette (Ctrl+P), search "Copy Context" — both entries should appear. +4. Manual: Click "Copy Collection Context". Verify clipboard content is a JSON array. +5. Manual: Try clicking "Copy Context" with no active note — should show error notice. +6. Plugin loads without errors: check Developer Console (Ctrl+Shift+I) for any JS errors. +</verification> + +<success_criteria> +- D-07: "Copy Context" button copies single-paper canonical index entry JSON to clipboard +- D-08: "Copy Collection Context" button copies filtered/all entries JSON array to clipboard +- AIC-02: User can generate a traceable context pack for a single paper from the plugin +- AIC-03: User can generate a collection-level context pack from the plugin +- AIC-04: Blocking factors and provenance are included in the output +- All 4 existing Quick Actions continue to function correctly +</success_criteria> + +<output> +After completion, create `.planning/phases/26-traceable-ai-context-packs/26-03-SUMMARY.md` +</output> diff --git a/.planning/phases/27-component-library/27-01-PLAN.md b/.planning/phases/27-component-library/27-01-PLAN.md new file mode 100644 index 00000000..72f50384 --- /dev/null +++ b/.planning/phases/27-component-library/27-01-PLAN.md @@ -0,0 +1,698 @@ +--- +phase: 27-component-library +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - paperforge/plugin/styles.css +autonomous: true +requirements: + - COMP-01 # Pure CSS/DOM dashboard visualizations + - COMP-02 # Obsidian CSS variable theming + - COMP-03 # Loading states, CSS transitions, responsive breakpoints + +must_haves: + truths: + - "Metric cards show large value + uppercase label with accent color left bar, optional progress bar variant, and CSS opacity transition on value change" + - "Lifecycle stepper renders 6 vertical steps with circles (border-radius:50%), human-readable labels (Imported/Indexed/PDF Ready/Fulltext Ready/Deep Read/AI Ready), and CSS ::before connecting lines between steps" + - "Health matrix renders as 2x2 CSS grid with 4 color-coded cells (PDF/OCR/Note/Asset), green/yellow/red status classes via Obsidian var(--color-*) variables, status icons, and hover tooltips" + - "Maturity gauge renders as 6-segment horizontal bar with filled segments colored and unfilled gray, current level number displayed below, blocking checks listed as bullet points when level < 6" + - "Bar chart renders horizontal bars with lifecycle-name label, proportional track/fill (width as percentage), count number, and smooth CSS transition: width 0.3s" + - "All 5 components show a loading skeleton shimmer (CSS @keyframes shimmer animation) when parent has .paperforge-loading class; empty state shows muted 'No data' message" + - "All colors use Obsidian CSS variables (var(--color-green), var(--color-yellow), var(--color-red), var(--color-cyan), var(--color-blue), var(--color-purple), var(--text-muted), var(--background-modifier-border)) and components respond to dark/light theme changes automatically" + artifacts: + - path: "paperforge/plugin/styles.css" + provides: "All component CSS including loading skeletons, themed via Obsidian CSS variables" + min_lines: 200 # new CSS content expected + key_links: + - from: "styles.css" + to: "Obsidian CSS variables" + via: "var(--color-green), var(--color-yellow), var(--color-red), var(--color-cyan), var(--color-blue), var(--color-purple), var(--text-muted), var(--background-modifier-border)" + - from: "styles.css .paperforge-loading" + to: "styles.css @keyframes paperforge-shimmer" + via: "CSS animation triggering loading skeleton on child elements" +--- + +<objective> +Create all CSS styling for 5 visualization components in `paperforge/plugin/styles.css`. Each component gets its own clearly commented section (Section 7 through Section 11+). All styling uses Obsidian CSS variables exclusively -- no hardcoded colors. Components are fully responsive across sidebar and full-width layouts. + +Purpose: Establish the visual contract that Phase 27-02 (JS render methods) will implement against. CSS class names are locked by CONTEXT.md decisions D-04 through D-23. +Output: `paperforge/plugin/styles.css` with 5 new sections appended after existing Section 6 (Setup Wizard Modal). +</objective> + +<execution_context> +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@C:/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> +@.planning/phases/27-component-library/27-CONTEXT.md +@paperforge/plugin/styles.css + +No prior plan SUMMARY references needed -- this is the first plan of Phase 27. +</context> + +<tasks> + +<task type="auto"> + <name>Task 1: Add loading skeleton CSS + enhanced metric card CSS</name> + <files>paperforge/plugin/styles.css</files> + + <read_first> + - paperforge/plugin/styles.css (existing file, to see pattern and append after Section 6) + - .planning/phases/27-component-library/27-CONTEXT.md (decisions D-04, D-05, D-06, D-24, D-25, D-26, D-27, D-29) + </read_first> + + <action> + Append to `paperforge/plugin/styles.css` after the last line (Section 6 ends at line 658). Add TWO new sections: + + **Section 7 -- Loading Skeleton & Empty States (D-24, D-25)** + + ```css + /* ========================================================================== + SECTION 7 — Loading Skeleton & Empty States + ========================================================================== */ + .paperforge-loading { + position: relative; + overflow: hidden; + } + + .paperforge-loading > * { + opacity: 0.6; + pointer-events: none; + } + + .paperforge-loading::after { + content: ''; + position: absolute; + inset: 0; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.08) 50%, + transparent 100% + ); + background-size: 200% 100%; + animation: paperforge-shimmer 1.5s ease-in-out infinite; + pointer-events: none; + } + + @keyframes paperforge-shimmer { + 0% { background-position: 200% 0; } + 100% { background-position: -200% 0; } + } + + /* Dark mode adjustment: lighter shimmer is slightly dimmer */ + .theme-dark .paperforge-loading::after { + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.04) 50%, + transparent 100% + ); + background-size: 200% 100%; + } + + .paperforge-empty-state { + text-align: center; + padding: 24px 12px; + color: var(--text-muted); + font-size: var(--font-ui-small); + font-style: italic; + } + ``` + + **Section 8 -- Metric Card (Enhanced, D-04, D-05, D-06)** + + The existing `.paperforge-metric-card` at lines 99-143 already has the basic structure. Replace the existing block with this enhanced version that adds the optional progress bar and value transition. IMPORTANT: Keep the existing `.paperforge-metrics` grid container (lines 93-97) unchanged. + + Replace the ENTIRE existing Section 2 block (lines 99 through 143 -- the `.paperforge-metric-card`, `.paperforge-metric-value`, `.paperforge-metric-label` rules) with this enhanced version: + + ```css + .paperforge-metric-card { + background: var(--background-secondary); + border: 1px solid var(--background-modifier-border); + border-radius: var(--radius-m); + padding: 16px 12px 14px; + text-align: center; + position: relative; + overflow: hidden; + transition: border-color 0.15s, opacity 0.3s; + } + + .paperforge-metric-card::before { + content: ''; + position: absolute; + left: 0; + top: 0; + bottom: 0; + width: 3px; + background: var(--metric-color, var(--interactive-accent)); + border-radius: 0 2px 2px 0; + } + + .paperforge-metric-card:hover { + border-color: var(--interactive-accent); + } + + .paperforge-metric-value { + font-size: clamp(18px, 4vw, 28px); + font-weight: var(--font-bold); + line-height: 1.1; + color: var(--text-normal); + letter-spacing: -0.5px; + transition: opacity 0.3s; + } + + .paperforge-metric-label { + font-size: var(--font-ui-smaller); + color: var(--text-muted); + margin-top: 4px; + text-transform: uppercase; + letter-spacing: 0.5px; + font-weight: var(--font-medium); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + /* Optional progress bar inside metric card (D-05) */ + .paperforge-metric-progress { + margin-top: 10px; + height: 4px; + border-radius: 2px; + background: var(--background-modifier-border); + overflow: hidden; + } + + .paperforge-metric-progress-fill { + height: 100%; + border-radius: 2px; + background: var(--metric-color, var(--interactive-accent)); + transition: width 0.3s; + } + ``` + + **DO NOT** touch the `.paperforge-metrics` grid container (lines 93-97). Leave it exactly as-is. Only replace the individual card/value/label rules (what was lines 99-143). + + </action> + + <acceptance_criteria> + - `styles.css` contains `@keyframes paperforge-shimmer` with `background-position` keyframes + - `styles.css` contains `.paperforge-loading::after` with the shimmer gradient + - `styles.css` contains `.paperforge-empty-state` with `var(--text-muted)`, `font-style: italic` + - `styles.css` contains `.paperforge-metric-card` with `transition: border-color 0.15s, opacity 0.3s` (D-06 opacity transition added) + - `styles.css` contains `.paperforge-metric-progress` and `.paperforge-metric-progress-fill` classes + - `styles.css` still contains `.paperforge-metrics` grid container class + - All colors use `var(--*)` Obsidian CSS variables — grep for hardcoded hex/rgb: zero matches outside `.paperforge-install-log` and `.theme-dark` override + - Shimmer animation duration is exactly `1.5s` + </acceptance_criteria> + + <verify> + <automated>python -c " +with open('paperforge/plugin/styles.css') as f: + c = f.read() +assert '@keyframes paperforge-shimmer' in c, 'Missing shimmer keyframes' +assert '.paperforge-loading::after' in c, 'Missing loading pseudo-element' +assert '.paperforge-empty-state' in c, 'Missing empty state' +assert '.paperforge-metric-card' in c, 'Missing metric card' +assert '.paperforge-metric-progress-fill' in c, 'Missing progress fill' +assert 'transition: border-color 0.15s, opacity 0.3s' in c, 'Missing opacity transition' +assert 'var(--text-muted)' in c, 'Hardcoded color detected' +print('Task 1 CSS verification: ALL CHECKS PASSED') +"</automated> + </verify> + + <done> + Loading skeleton shimmer CSS is defined as @keyframes paperforge-shimmer with 1.5s animation. Enhanced metric card has opacity 0.3s transition on value change and optional progress bar sub-element. Empty state has muted italic styling. All use Obsidian CSS variables. + </done> +</task> + +<task type="auto"> + <name>Task 2: Add lifecycle stepper CSS + health matrix CSS</name> + <files>paperforge/plugin/styles.css</files> + + <read_first> + - paperforge/plugin/styles.css (output after Task 1, to append after Section 8) + - .planning/phases/27-component-library/27-CONTEXT.md (decisions D-07 through D-16, D-26, D-27, D-29) + </read_first> + + <action> + Append to `paperforge/plugin/styles.css` after Section 8 (Metric Card). Add TWO sections: + + **Section 9 -- Lifecycle Stepper (D-07 through D-11)** + + ```css + /* ========================================================================== + SECTION 9 — Lifecycle Stepper + ========================================================================== */ + .paperforge-lifecycle-stepper { + display: flex; + flex-direction: column; + gap: 0; + padding: 8px 0; + position: relative; + } + + .paperforge-lifecycle-stepper .step { + display: flex; + align-items: center; + gap: 12px; + padding: 10px 12px; + position: relative; + min-height: 36px; + } + + .paperforge-lifecycle-stepper .step-indicator { + width: 24px; + height: 24px; + border-radius: 50%; + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: center; + font-size: 12px; + font-weight: var(--font-bold); + position: relative; + z-index: 1; + background: var(--background-modifier-border); + color: var(--text-muted); + border: 2px solid var(--background-modifier-border); + transition: background 0.2s, border-color 0.2s, color 0.2s; + } + + /* Connecting line between steps via ::before on each step (D-09) */ + .paperforge-lifecycle-stepper .step::before { + content: ''; + position: absolute; + left: 22px; /* centers under the 24px indicator circle */ + top: 32px; /* below the indicator */ + bottom: -8px; + width: 2px; + background: var(--background-modifier-border); + z-index: 0; + } + + .paperforge-lifecycle-stepper .step:last-child::before { + display: none; + } + + .paperforge-lifecycle-stepper .step-label { + font-size: var(--font-ui-small); + color: var(--text-muted); + font-weight: var(--font-medium); + line-height: 1.3; + } + + /* -- State: completed (D-10) -- */ + .paperforge-lifecycle-stepper .step.completed .step-indicator { + background: var(--color-green); + border-color: var(--color-green); + color: var(--text-on-accent); + } + + .paperforge-lifecycle-stepper .step.completed .step-indicator::after { + content: '\u2713'; /* checkmark */ + } + + .paperforge-lifecycle-stepper .step.completed .step-label { + color: var(--text-normal); + } + + .paperforge-lifecycle-stepper .step.completed::before { + background: var(--color-green); + } + + /* -- State: current (D-10) -- */ + .paperforge-lifecycle-stepper .step.current .step-indicator { + background: var(--interactive-accent); + border-color: var(--interactive-accent); + color: var(--text-on-accent); + animation: paperforge-step-pulse 2s ease-in-out infinite; + } + + @keyframes paperforge-step-pulse { + 0%, 100% { box-shadow: 0 0 0 0 rgba(255,255,255,0.3); } + 50% { box-shadow: 0 0 0 6px rgba(255,255,255,0); } + } + + .paperforge-lifecycle-stepper .step.current .step-label { + color: var(--text-normal); + font-weight: var(--font-semibold); + } + + /* -- State: pending (D-10) -- */ + .paperforge-lifecycle-stepper .step.pending .step-indicator { + opacity: 0.4; + } + + .paperforge-lifecycle-stepper .step.pending .step-label { + opacity: 0.5; + } + ``` + + **Section 10 -- Health Matrix (D-12 through D-16)** + + ```css + /* ========================================================================== + SECTION 10 — Health Matrix + ========================================================================== */ + .paperforge-health-matrix { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; + } + + .paperforge-health-cell { + background: var(--background-secondary); + border: 1px solid var(--background-modifier-border); + border-radius: var(--radius-m); + padding: 14px 12px; + display: flex; + flex-direction: column; + align-items: center; + gap: 6px; + cursor: help; + transition: border-color 0.15s; + position: relative; + } + + .paperforge-health-cell:hover { + border-color: var(--interactive-accent); + } + + .paperforge-health-cell-icon { + font-size: 18px; + line-height: 1; + } + + .paperforge-health-cell-label { + font-size: var(--font-ui-smaller); + font-weight: var(--font-medium); + color: var(--text-normal); + text-transform: uppercase; + letter-spacing: 0.3px; + } + + /* Status coloring (D-14) */ + .paperforge-health-cell.ok { + border-left: 3px solid var(--color-green); + } + + .paperforge-health-cell.ok .paperforge-health-cell-icon { + color: var(--color-green); + } + + .paperforge-health-cell.warn { + border-left: 3px solid var(--color-yellow); + } + + .paperforge-health-cell.warn .paperforge-health-cell-icon { + color: var(--color-yellow); + } + + .paperforge-health-cell.fail { + border-left: 3px solid var(--color-red); + } + + .paperforge-health-cell.fail .paperforge-health-cell-icon { + color: var(--color-red); + } + + /* Tooltip on hover (D-16) */ + .paperforge-health-cell[title]:hover::after { + content: attr(title); + position: absolute; + bottom: calc(100% + 6px); + left: 50%; + transform: translateX(-50%); + background: var(--background-modifier-hover); + color: var(--text-normal); + font-size: 11px; + padding: 4px 10px; + border-radius: var(--radius-s); + white-space: nowrap; + pointer-events: none; + z-index: 100; + border: 1px solid var(--background-modifier-border); + box-shadow: 0 2px 6px rgba(0,0,0,0.1); + } + + .paperforge-health-cell[title]:hover::before { + content: ''; + position: absolute; + bottom: calc(100% + 2px); + left: 50%; + transform: translateX(-50%); + border: 6px solid transparent; + border-top-color: var(--background-modifier-border); + pointer-events: none; + z-index: 100; + } + ``` + + </action> + + <acceptance_criteria> + - `styles.css` contains `.paperforge-lifecycle-stepper` with `flex-direction: column` + - `styles.css` contains `.step-indicator` with `border-radius: 50%` and `width: 24px; height: 24px` + - `styles.css` contains `.step::before` with positioned connecting line (`top: 32px`, `width: 2px`) + - `styles.css` contains `.step.completed` rules referencing `var(--color-green)` + - `styles.css` contains `.step.current` rules referencing `var(--interactive-accent)` and `@keyframes paperforge-step-pulse` + - `styles.css` contains `.step.pending` with `opacity: 0.4` + - `styles.css` contains `.paperforge-health-matrix` with `grid-template-columns: 1fr 1fr` + - `styles.css` contains `.paperforge-health-cell.ok` with `var(--color-green)`, `.warn` with `var(--color-yellow)`, `.fail` with `var(--color-red)` + - `styles.css` contains hover tooltip via `[title]:hover::after` with `attr(title)` content + - All colors use `var(--*)` — zero hardcoded hex/rgb values + </acceptance_criteria> + + <verify> + <automated>python -c " +with open('paperforge/plugin/styles.css') as f: + c = f.read() +assert '.paperforge-lifecycle-stepper' in c, 'Missing lifecycle stepper' +assert '.step-indicator' in c and 'border-radius: 50%' in c, 'Missing step indicator circle' +assert '.step.completed' in c and 'var(--color-green)' in c, 'Missing completed state' +assert '.step.current' in c and 'var(--interactive-accent)' in c, 'Missing current state' +assert '@keyframes paperforge-step-pulse' in c, 'Missing pulse animation' +assert '.step.pending' in c, 'Missing pending state' +assert '.paperforge-health-matrix' in c, 'Missing health matrix' +assert 'grid-template-columns: 1fr 1fr' in c, 'Wrong grid layout' +assert '.paperforge-health-cell.ok' in c, 'Missing ok class' +assert '.paperforge-health-cell.warn' in c, 'Missing warn class' +assert '.paperforge-health-cell.fail' in c, 'Missing fail class' +assert 'attr(title)' in c, 'Missing tooltip via attr' +print('Task 2 CSS verification: ALL CHECKS PASSED') +"</automated> + </verify> + + <done> + Lifecycle stepper renders 6 vertical steps with border-radius circles, ::before connecting lines, and completed/current/pending CSS class states. Health matrix renders as 2x2 grid with color-coded cells using Obsidian CSS variables and hover tooltips. Pulse animation defined for current step. + </done> +</task> + +<task type="auto"> + <name>Task 3: Add maturity gauge CSS + bar chart CSS</name> + <files>paperforge/plugin/styles.css</files> + + <read_first> + - paperforge/plugin/styles.css (output after Task 2, to append after Section 10) + - .planning/phases/27-component-library/27-CONTEXT.md (decisions D-17 through D-23, D-26, D-27, D-29) + </read_first> + + <action> + Append to `paperforge/plugin/styles.css` after Section 10 (Health Matrix). Add TWO sections: + + **Section 11 -- Maturity Gauge (D-17 through D-20)** + + ```css + /* ========================================================================== + SECTION 11 — Maturity Gauge + ========================================================================== */ + .paperforge-maturity-gauge { + display: flex; + flex-direction: column; + gap: 12px; + } + + .paperforge-maturity-gauge .gauge-track { + display: flex; + gap: 4px; + height: 16px; + border-radius: 8px; + overflow: hidden; + background: var(--background-modifier-border); + } + + .paperforge-maturity-gauge .gauge-segment { + flex: 1; + height: 100%; + border-radius: 2px; + background: var(--background-modifier-border); + transition: background 0.3s; + } + + .paperforge-maturity-gauge .gauge-segment.filled { + background: var(--interactive-accent); + } + + /* Gradient variant: each filled segment gets progressively richer color (the agent's discretion) */ + .paperforge-maturity-gauge .gauge-segment.filled.level-1 { background: var(--color-cyan); } + .paperforge-maturity-gauge .gauge-segment.filled.level-2 { background: var(--color-blue); } + .paperforge-maturity-gauge .gauge-segment.filled.level-3 { background: var(--color-purple); } + .paperforge-maturity-gauge .gauge-segment.filled.level-4 { background: var(--color-green); } + .paperforge-maturity-gauge .gauge-segment.filled.level-5 { background: var(--color-yellow); } + .paperforge-maturity-gauge .gauge-segment.filled.level-6 { background: var(--color-red); } + + .paperforge-maturity-gauge .gauge-level { + font-size: var(--font-ui-medium); + font-weight: var(--font-bold); + color: var(--text-normal); + text-align: center; + } + + .paperforge-maturity-gauge .gauge-blockers { + list-style: disc; + padding-left: 20px; + margin: 4px 0 0; + } + + .paperforge-maturity-gauge .gauge-blockers li { + font-size: var(--font-ui-smaller); + color: var(--text-muted); + line-height: 1.5; + } + ``` + + **Section 12 -- Bar Chart (D-21 through D-23)** + + ```css + /* ========================================================================== + SECTION 12 — Bar Chart + ========================================================================== */ + .paperforge-bar-chart { + display: flex; + flex-direction: column; + gap: 8px; + padding: 4px 0; + } + + .paperforge-bar-chart .bar-row { + display: flex; + align-items: center; + gap: 10px; + } + + .paperforge-bar-chart .bar-label { + font-size: var(--font-ui-smaller); + color: var(--text-muted); + flex: 0 0 90px; + text-align: right; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + font-weight: var(--font-medium); + } + + .paperforge-bar-chart .bar-track { + flex: 1; + height: 20px; + border-radius: 4px; + background: var(--background-modifier-border); + overflow: hidden; + position: relative; + } + + .paperforge-bar-chart .bar-fill { + height: 100%; + border-radius: 4px; + background: var(--interactive-accent); + transition: width 0.3s; + min-width: 2px; + } + + .paperforge-bar-chart .bar-count { + font-size: var(--font-ui-smaller); + font-weight: var(--font-bold); + color: var(--text-normal); + flex: 0 0 30px; + text-align: left; + font-variant-numeric: tabular-nums; + } + + /* Color variants for bar fills based on lifecycle stage */ + .paperforge-bar-chart .bar-fill.stage-imported { background: var(--color-cyan); } + .paperforge-bar-chart .bar-fill.stage-indexed { background: var(--color-blue); } + .paperforge-bar-chart .bar-fill.stage-pdf-ready { background: var(--color-purple); } + .paperforge-bar-chart .bar-fill.stage-fulltext-ready { background: var(--color-green); } + .paperforge-bar-chart .bar-fill.stage-deep-read { background: var(--color-yellow); } + .paperforge-bar-chart .bar-fill.stage-ai-ready { background: var(--color-red); } + ``` + + </action> + + <acceptance_criteria> + - `styles.css` contains `.paperforge-maturity-gauge` with `flex-direction: column` + - `styles.css` contains `.gauge-segment.filled` with `var(--interactive-accent)` background + - `styles.css` contains `.gauge-segment.filled.level-1` through `.level-6` with distinct var(--color-*) values + - `styles.css` contains `.gauge-blockers` as `list-style: disc` with `padding-left: 20px` + - `styles.css` contains `.paperforge-bar-chart` with `flex-direction: column` + - `styles.css` contains `.bar-row` with `display: flex; align-items: center` + - `styles.css` contains `.bar-fill` with `transition: width 0.3s` (D-23) + - `styles.css` contains `.bar-fill.stage-imported` through `.stage-ai-ready` color variant classes + - All colors use `var(--*)` — zero hardcoded hex/rgb values + </acceptance_criteria> + + <verify> + <automated>python -c " +with open('paperforge/plugin/styles.css') as f: + c = f.read() +assert '.paperforge-maturity-gauge' in c, 'Missing maturity gauge' +assert '.gauge-segment.filled' in c, 'Missing filled segment' +assert '.gauge-segment.filled.level-1' in c, 'Missing level-1 color' +assert '.gauge-segment.filled.level-6' in c, 'Missing level-6 color' +assert '.gauge-blockers' in c, 'Missing blockers list' +assert '.paperforge-bar-chart' in c, 'Missing bar chart' +assert '.bar-row' in c, 'Missing bar row' +assert '.bar-fill' in c, 'Missing bar fill' +assert 'transition: width 0.3s' in c, 'Missing width transition' +assert '.bar-fill.stage-imported' in c, 'Missing stage-imported color' +assert '.bar-fill.stage-ai-ready' in c, 'Missing stage-ai-ready color' +print('Task 3 CSS verification: ALL CHECKS PASSED') +"</automated> + </verify> + + <done> + Maturity gauge renders as segmented progress bar with 6 level-specific color classes, level number display, and blocking checks list. Bar chart renders horizontal bars with lifecycle stage color variants, transition: width 0.3s. All components use Obsidian CSS variables. + </done> +</task> + +</tasks> + +<verification> +1. All 5 component CSS rule sets exist in styles.css (metric card, lifecycle stepper, health matrix, maturity gauge, bar chart) +2. Loading skeleton shimmer keyframes defined +3. Empty state defined +4. @keyframes paperforge-shimmer and @keyframes paperforge-step-pulse both defined +5. All CSS uses var(--*) for colors — grep for `#[0-9a-f]` or `rgb(` returns zero matches outside exceptions (`.paperforge-install-log` hardcoded color is pre-existing) +6. Section header comments follow existing pattern: `/* ===== SECTION N — Name ===== */` +7. No hardcoded colors outside the two allowed exceptions: `.paperforge-install-log` pre-existing dark background, and `.theme-dark .paperforge-loading::after` +</verification> + +<success_criteria> +- [ ] styles.css has all 12 sections (6 existing + 6 new: Loading Skeleton, Metric Card enhanced, Lifecycle Stepper, Health Matrix, Maturity Gauge, Bar Chart) +- [ ] Each new section has a header comment matching `/* ===== SECTION N — Name ===== */` pattern +- [ ] All component class names match CONTEXT.md decisions (D-04 through D-23) +- [ ] All colors reference Obsidian CSS variables +- [ ] Loading skeleton is a reusable class (`.paperforge-loading`) that any component can use +- [ ] Empty state is a reusable class (`.paperforge-empty-state`) +- [ ] CSS transitions defined for metric value opacity, bar-fill width, and gauge segment color +- [ ] Health matrix tooltip implemented via `[title]:hover::after` +- [ ] Lifecycle stepper connecting lines via `::before` pseudo-element +- [ ] No npm dependencies, no external CSS frameworks, no JS chart libraries (D-03) +</success_criteria> + +<output> +After completion, create `.planning/phases/27-component-library/27-01-SUMMARY.md` +</output> diff --git a/.planning/phases/27-component-library/27-02-PLAN.md b/.planning/phases/27-component-library/27-02-PLAN.md new file mode 100644 index 00000000..26c56a24 --- /dev/null +++ b/.planning/phases/27-component-library/27-02-PLAN.md @@ -0,0 +1,475 @@ +--- +phase: 27-component-library +plan: 02 +type: execute +wave: 2 +depends_on: + - 27-01 # Component CSS must exist — JS methods reference CSS class names +files_modified: + - paperforge/plugin/main.js +autonomous: true +requirements: + - COMP-01 # Pure CSS/DOM dashboard visualizations + - COMP-03 # Loading states, CSS transitions, responsive breakpoints + +must_haves: + truths: + - "PaperForgeStatusView has a _renderLifecycleStepper(lifecycle, currentStage) method producing a div.paperforge-lifecycle-stepper with 6 step divs, each with .step-indicator and .step-label, with appropriate .completed/.current/.pending classes applied" + - "PaperForgeStatusView has a _renderHealthMatrix(health) method producing a div.paperforge-health-matrix with 4 div.paperforge-health-cell children (PDF/OCR/Note/Asset), each with .ok/.warn/.fail class, status icon (unicode), and title attribute for tooltip" + - "PaperForgeStatusView has a _renderMaturityGauge(maturityLevel, blockingChecks) method producing a div.paperforge-maturity-gauge with 6 segment divs (filled/unfilled), level number, and optional blocking checks list" + - "PaperForgeStatusView has a _renderBarChart(lifecycleCounts) method producing a div.paperforge-bar-chart with div.bar-row children, each containing .bar-label, .bar-track > .bar-fill, and .bar-count" + - "All 4 new render methods (and enhanced _renderStats) accept null/undefined data and render a loading skeleton container (add .paperforge-loading class to parent) instead of crashing" + - "Enhanced _renderStats() supports optional progress sub-bar on any metric card via a _buildMetricBar() helper" + artifacts: + - path: "paperforge/plugin/main.js" + provides: "New render methods on PaperForgeStatusView: _renderLifecycleStepper, _renderHealthMatrix, _renderMaturityGauge, _renderBarChart, plus enhanced _renderStats and _buildMetricBar helper" + min_lines: 100 # new JS content expected + key_links: + - from: "main.js PaperForgeStatusView._renderLifecycleStepper" + to: "styles.css .paperforge-lifecycle-stepper, .step, .step-indicator, .step-label, .completed, .current, .pending" + via: "createEl() cls attribute matching CSS classes" + - from: "main.js PaperForgeStatusView._renderHealthMatrix" + to: "styles.css .paperforge-health-matrix, .health-cell, .ok, .warn, .fail" + via: "createEl() cls attribute and addClass() calls" + - from: "main.js PaperForgeStatusView._renderMaturityGauge" + to: "styles.css .paperforge-maturity-gauge, .gauge-segment, .filled, .level-*, .gauge-blockers" + via: "createEl() cls attribute matching CSS classes" + - from: "main.js PaperForgeStatusView._renderBarChart" + to: "styles.css .paperforge-bar-chart, .bar-row, .bar-label, .bar-track, .bar-fill, .bar-count" + via: "createEl() cls attribute matching CSS classes" +--- + +<objective> +Implement 5 render methods on `PaperForgeStatusView` class in `paperforge/plugin/main.js` that produce DOM elements matching the CSS class names from Plan 27-01. All methods use `createEl()` DOM API -- no JSX, no innerHTML (D-01). All methods gracefully handle null/undefined data by rendering a loading skeleton (D-24). No new JS files -- all methods live on the existing PaperForgeStatusView class (D-28). + +Purpose: Provide the render methods that Phase 28 (Dashboard Shell & Context Detection) and Phase 29/30 will call to populate dashboard views with concrete data. +Output: `paperforge/plugin/main.js` with 4 new methods and 1 enhanced method + 1 helper. +</objective> + +<execution_context> +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@C:/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> +@.planning/phases/27-component-library/27-CONTEXT.md +@paperforge/plugin/main.js + +No prior plan SUMMARY references needed -- this is the second plan of Phase 27. +</context> + +<tasks> + +<task type="auto"> + <name>Task 1: Add loading skeleton utility + enhanced _renderStats() + _buildMetricBar() helper</name> + <files>paperforge/plugin/main.js</files> + + <read_first> + - paperforge/plugin/main.js (entire file, especially the PaperForgeStatusView class at lines 210-638, the existing _renderStats() at lines 378-392, and _buildPanel() at lines 232-280) + - .planning/phases/27-component-library/27-CONTEXT.md (decisions D-01, D-04, D-05, D-06, D-24, D-25, D-28) + </read_first> + + <action> + Add the following to the `PaperForgeStatusView` class (between lines 210 and 638). Insert these methods AFTER the existing `_renderOcr(d)` method (which ends around line 461) and BEFORE the existing `_runAction(a, card)` method (which starts around line 463). + + **Method 1: `_renderSkeleton(container)` — utility that adds loading skeleton to any container (D-24)** + + ```javascript + /* ── Loading Skeleton Utility (D-24) ── */ + _renderSkeleton(container) { + container.addClass('paperforge-loading'); + } + + /* ── Empty State Utility (D-25) ── */ + _renderEmptyState(container, message) { + container.createEl('div', { + cls: 'paperforge-empty-state', + text: message || 'No data', + }); + } + ``` + + **Method 2: `_buildMetricBar(card, value, max)` — adds optional progress bar to a metric card (D-05)** + + ```javascript + /* ── Metric Progress Bar Helper (D-05) ── */ + _buildMetricBar(card, value, max) { + if (max <= 0) return; + const pct = Math.min(100, (value / max) * 100); + const bar = card.createEl('div', { cls: 'paperforge-metric-progress' }); + bar.createEl('div', { + cls: 'paperforge-metric-progress-fill', + attr: { style: `width:${pct.toFixed(1)}%` }, + }); + } + ``` + + **Method 3: Enhanced `_renderStats(d)` — adds loading skeleton support and optional progress bar (D-04, D-05, D-06)** + + Replace the EXISTING `_renderStats(d)` method (current lines 378-392) with this enhanced version: + + ```javascript + /* ── Metric Cards (Enhanced D-04, D-05, D-06) ── */ + _renderStats(d) { + this._versionBadge.setText(d.version ? 'v' + d.version : 'v\u2014'); + + if (!d || typeof d.total_papers === 'undefined') { + this._renderSkeleton(this._metricsEl); + return; + } + + this._metricsEl.removeClass('paperforge-loading'); + + const totalPapers = d.total_papers || 0; + const totalFormal = d.formal_notes || 0; + + const metrics = [ + { value: totalPapers, label: 'Papers', color: 'var(--color-cyan)', barMax: 0 }, + { value: totalFormal, label: 'Formal Notes', color: 'var(--color-blue)', barMax: totalPapers }, + { value: d.exports || 0, label: 'Exports', color: 'var(--color-purple)', barMax: 0 }, + ]; + for (const m of metrics) { + const card = this._metricsEl.createEl('div', { cls: 'paperforge-metric-card' }); + card.style.setProperty('--metric-color', m.color); + card.createEl('div', { cls: 'paperforge-metric-value', text: m.value?.toString() || '\u2014' }); + card.createEl('div', { cls: 'paperforge-metric-label', text: m.label }); + if (m.barMax > 0) { + this._buildMetricBar(card, m.value, m.barMax); + } + } + } + ``` + + Key changes from existing: + - Added null check at top: if `d` is null/undefined or `d.total_papers` is undefined, render skeleton instead + - Added `removeClass('paperforge-loading')` when data is valid + - Changed Notes label from "Notes" to "Formal Notes" for clarity + - Added `barMax` field to metric config -- when > 0, `_buildMetricBar` is called + - Formal notes card gets a progress bar showing `formal_notes / total_papers` as fill percentage + </action> + + <acceptance_criteria> + - `main.js` PaperForgeStatusView class has a `_renderSkeleton(container)` method that calls `container.addClass('paperforge-loading')` + - `main.js` PaperForgeStatusView class has a `_renderEmptyState(container, message)` method that creates a `div.paperforge-empty-state` + - `main.js` PaperForgeStatusView class has a `_buildMetricBar(card, value, max)` method that creates `div.paperforge-metric-progress` with child `div.paperforge-metric-progress-fill` + - `main.js` enhanced `_renderStats(d)` returns early with skeleton when `d` is null/undefined + - `main.js` enhanced `_renderStats(d)` has the label "Formal Notes" instead of "Notes" + - `main.js` enhanced `_renderStats(d)` calls `_buildMetricBar()` for the notes metric + - All methods are defined as class methods (using proper `methodName() {` syntax, not arrow functions) + - grep: `_renderSkeleton\b` exists, `_renderEmptyState\b` exists, `_buildMetricBar\b` exists + - grep: `'Formal Notes'` appears in main.js + </acceptance_criteria> + + <verify> + <automated>python -c " +with open('paperforge/plugin/main.js') as f: + c = f.read() +assert '_renderSkeleton' in c, 'Missing _renderSkeleton method' +assert '_renderEmptyState' in c, 'Missing _renderEmptyState method' +assert '_buildMetricBar' in c, 'Missing _buildMetricBar method' +assert \"'Formal Notes'\" in c, 'Missing updated label Formal Notes' +assert 'paperforge-metric-progress' in c, 'Missing progress bar class reference' +assert 'paperforge-loading' in c, 'Missing loading class reference' +assert 'addClass' in c and 'removeClass' in c, 'Missing class management calls' +print('Task 1 JS verification: ALL CHECKS PASSED') +"</automated> + </verify> + + <done> + Enhanced _renderStats() handles null data via skeleton loading state. _renderSkeleton() and _renderEmptyState() are reusable utilities. _buildMetricBar() adds optional progress bar to metric cards. Formal Notes card shows coverage ratio as a progress bar. + </done> +</task> + +<task type="auto"> + <name>Task 2: Add _renderLifecycleStepper() + _renderHealthMatrix() methods</name> + <files>paperforge/plugin/main.js</files> + + <read_first> + - paperforge/plugin/main.js (PaperForgeStatusView class methods, especially around the existing _renderStats/_renderOcr area) + - .planning/phases/27-component-library/27-CONTEXT.md (decisions D-07 through D-16, D-24, D-25, D-26, D-28) + </read_first> + + <action> + Add the following TWO methods to the PaperForgeStatusView class. Insert them immediately after the `_buildMetricBar` method added in Task 1, continuing the pattern of utility/render methods before `_runAction`. + + **Method 4: `_renderLifecycleStepper(container, lifecycle, currentStage)` — renders 6-step vertical lifecycle indicator (D-07 through D-11, D-24)** + + ```javascript + /* ── Lifecycle Stepper (D-07 through D-11) ── */ + _renderLifecycleStepper(container, lifecycle, currentStage) { + if (!lifecycle || !currentStage) { + this._renderSkeleton(container); + return; + } + + const stages = [ + { key: 'imported', label: 'Imported' }, + { key: 'indexed', label: 'Indexed' }, + { key: 'pdf_ready', label: 'PDF Ready' }, + { key: 'fulltext_ready', label: 'Fulltext Ready' }, + { key: 'deep_read', label: 'Deep Read' }, + { key: 'ai_ready', label: 'AI Ready' }, + ]; + + const stepper = container.createEl('div', { cls: 'paperforge-lifecycle-stepper' }); + let foundCurrent = false; + + for (const stage of stages) { + const step = stepper.createEl('div', { cls: 'step' }); + step.createEl('div', { cls: 'step-indicator' }); + step.createEl('div', { cls: 'step-label', text: stage.label }); + + if (stage.key === currentStage) { + step.addClass('current'); + foundCurrent = true; + } else if (!foundCurrent) { + // All stages before current are completed + step.addClass('completed'); + } else { + step.addClass('pending'); + } + } + } + ``` + + **Method 5: `_renderHealthMatrix(container, health)` — renders 2x2 health grid (D-12 through D-16, D-24)** + + ```javascript + /* ── Health Matrix (D-12 through D-16) ── */ + _renderHealthMatrix(container, health) { + if (!health) { + this._renderSkeleton(container); + return; + } + + const dimensions = [ + { key: 'pdf_health', label: 'PDF Health', iconOk: '\u2713', iconWarn: '\u26A0', iconFail: '\u2717' }, + { key: 'ocr_health', label: 'OCR Health', iconOk: '\u2713', iconWarn: '\u26A0', iconFail: '\u2717' }, + { key: 'note_health', label: 'Note Health', iconOk: '\u2713', iconWarn: '\u26A0', iconFail: '\u2717' }, + { key: 'asset_health', label: 'Asset Health', iconOk: '\u2713', iconWarn: '\u26A0', iconFail: '\u2717' }, + ]; + + const matrix = container.createEl('div', { cls: 'paperforge-health-matrix' }); + + for (const dim of dimensions) { + const status = health[dim.key] || 'healthy'; + const cell = matrix.createEl('div', { cls: 'paperforge-health-cell' }); + + let icon, statusClass, tooltip; + if (status === 'healthy' || status === 'ok') { + icon = dim.iconOk; + statusClass = 'ok'; + tooltip = `${dim.label}: OK`; + } else if (status === 'warn' || status === 'warning' || status === 'degraded') { + icon = dim.iconWarn; + statusClass = 'warn'; + tooltip = `${dim.label}: Needs Attention`; + } else { + icon = dim.iconFail; + statusClass = 'fail'; + tooltip = `${dim.label}: Failed`; + } + + cell.addClass(statusClass); + cell.setAttribute('title', tooltip); + cell.createEl('div', { cls: 'paperforge-health-cell-icon', text: icon }); + cell.createEl('div', { cls: 'paperforge-health-cell-label', text: dim.label }); + } + } + ``` + </action> + + <acceptance_criteria> + - `main.js` contains `_renderLifecycleStepper(container, lifecycle, currentStage)` method + - Method renders exactly 6 stages matching D-11 labels: Imported, Indexed, PDF Ready, Fulltext Ready, Deep Read, AI Ready + - Each step has `div.step` with children `div.step-indicator` and `div.step-label` + - `addClass('current')` for matching stage, `addClass('completed')` for prior stages, `addClass('pending')` for later stages + - Returns early with `_renderSkeleton(container)` when lifecycle or currentStage is null/undefined + - `main.js` contains `_renderHealthMatrix(container, health)` method + - Method creates `div.paperforge-health-matrix` with 4 `div.paperforge-health-cell` children + - Each cell gets `.ok`/`.warn`/`.fail` class based on health status value + - Each cell has `title` attribute set (for CSS tooltip per D-16) + - Each cell has icon div + label div + - Returns early with `_renderSkeleton(container)` when health is null/undefined + - grep: `_renderLifecycleStepper` exists, `_renderHealthMatrix` exists + - grep: `step-indicator` appears, `step-label` appears + - grep: `paperforge-health-cell` appears + </acceptance_criteria> + + <verify> + <automated>python -c " +with open('paperforge/plugin/main.js') as f: + c = f.read() +assert '_renderLifecycleStepper' in c, 'Missing lifecycle stepper method' +assert '_renderHealthMatrix' in c, 'Missing health matrix method' +assert \"'Imported'\" in c and \"'AI Ready'\" in c, 'Missing stage labels' +assert \"'step-indicator'\" in c, 'Missing step-indicator class' +assert \"'step-label'\" in c, 'Missing step-label class' +assert \"addClass('current')\" in c, 'Missing current state' +assert \"addClass('completed')\" in c, 'Missing completed state' +assert \"addClass('pending')\" in c, 'Missing pending state' +assert \"'paperforge-health-matrix'\" in c, 'Missing health matrix class' +assert \"'paperforge-health-cell'\" in c, 'Missing health cell class' +assert 'setAttribute' in c and \"'title'\" in c, 'Missing title attribute for tooltip' +assert \"addClass('ok')\" in c and \"addClass('fail')\" in c, 'Missing status classes' +print('Task 2 JS verification: ALL CHECKS PASSED') +"</automated> + </verify> + + <done> + _renderLifecycleStepper renders 6-stage vertical stepper with correct completed/current/pending classes. _renderHealthMatrix renders 2x2 grid with color-coded status cells and tooltip titles. Both methods gracefully handle null data via skeleton loading. + </done> +</task> + +<task type="auto"> + <name>Task 3: Add _renderMaturityGauge() + _renderBarChart() methods</name> + <files>paperforge/plugin/main.js</files> + + <read_first> + - paperforge/plugin/main.js (PaperForgeStatusView class methods area) + - .planning/phases/27-component-library/27-CONTEXT.md (decisions D-17 through D-25, D-28) + </read_first> + + <action> + Add the following TWO methods to the PaperForgeStatusView class. Insert them immediately after the `_renderHealthMatrix` method added in Task 2. + + **Method 6: `_renderMaturityGauge(container, maturityLevel, blockingChecks)` — renders 6-segment gauge (D-17 through D-20, D-24, D-25)** + + ```javascript + /* ── Maturity Gauge (D-17 through D-20) ── */ + _renderMaturityGauge(container, maturityLevel, blockingChecks) { + if (maturityLevel == null || maturityLevel === undefined) { + this._renderSkeleton(container); + return; + } + + const gauge = container.createEl('div', { cls: 'paperforge-maturity-gauge' }); + const track = gauge.createEl('div', { cls: 'gauge-track' }); + const currentLevel = Math.max(1, Math.min(6, Math.round(maturityLevel))); + + for (let i = 1; i <= 6; i++) { + const seg = track.createEl('div', { cls: 'gauge-segment' }); + if (i <= currentLevel) { + seg.addClass('filled'); + seg.addClass(`level-${i}`); + } + } + + gauge.createEl('div', { cls: 'gauge-level', text: `Level ${currentLevel} / 6` }); + + if (currentLevel < 6 && blockingChecks && blockingChecks.length > 0) { + const list = gauge.createEl('ul', { cls: 'gauge-blockers' }); + for (const check of blockingChecks) { + list.createEl('li', { text: check }); + } + } + } + ``` + + **Method 7: `_renderBarChart(container, lifecycleCounts)` — renders horizontal bars from lifecycle counts (D-21 through D-23, D-24, D-25)** + + ```javascript + /* ── Bar Chart (D-21 through D-23) ── */ + _renderBarChart(container, lifecycleCounts) { + if (!lifecycleCounts || Object.keys(lifecycleCounts).length === 0) { + this._renderEmptyState(container, 'No lifecycle data'); + return; + } + + const stages = [ + { key: 'imported', label: 'Imported', cls: 'stage-imported' }, + { key: 'indexed', label: 'Indexed', cls: 'stage-indexed' }, + { key: 'pdf_ready', label: 'PDF Ready', cls: 'stage-pdf-ready' }, + { key: 'fulltext_ready', label: 'Fulltext Ready', cls: 'stage-fulltext-ready' }, + { key: 'deep_read', label: 'Deep Read', cls: 'stage-deep-read' }, + { key: 'ai_ready', label: 'AI Ready', cls: 'stage-ai-ready' }, + ]; + + const chart = container.createEl('div', { cls: 'paperforge-bar-chart' }); + const maxCount = Math.max(1, ...stages.map(s => lifecycleCounts[s.key] || 0)); + + for (const stage of stages) { + const count = lifecycleCounts[stage.key] || 0; + const pct = (count / maxCount) * 100; + + const row = chart.createEl('div', { cls: 'bar-row' }); + row.createEl('div', { cls: 'bar-label', text: stage.label }); + const track = row.createEl('div', { cls: 'bar-track' }); + const fill = track.createEl('div', { + cls: `bar-fill ${stage.cls}`, + attr: { style: `width:${pct.toFixed(1)}%` }, + }); + row.createEl('div', { cls: 'bar-count', text: count.toString() }); + } + } + ``` + </action> + + <acceptance_criteria> + - `main.js` contains `_renderMaturityGauge(container, maturityLevel, blockingChecks)` method + - Method creates `div.paperforge-maturity-gauge` with child `div.gauge-track` + - Creates exactly 6 `div.gauge-segment` children, filled segments get `.filled` and `.level-N` classes + - Creates `div.gauge-level` with text "Level N / 6" + - Creates `ul.gauge-blockers` with `li` children only when currentLevel < 6 and blockingChecks is non-empty + - Returns early with `_renderSkeleton(container)` when maturityLevel is null/undefined + - `main.js` contains `_renderBarChart(container, lifecycleCounts)` method + - Method creates `div.paperforge-bar-chart` with `div.bar-row` children + - Each bar-row has child: `div.bar-label`, `div.bar-track` > `div.bar-fill`, `div.bar-count` + - Bar fill widths computed as percentages of max count + - Returns early with `_renderEmptyState(container, ...)` when lifecycleCounts is null/empty + - grep: `_renderMaturityGauge` exists, `_renderBarChart` exists + - grep: `gauge-segment` appears, `bar-fill` appears + - grep: `gauge-blockers` appears, `bar-count` appears + </acceptance_criteria> + + <verify> + <automated>python -c " +with open('paperforge/plugin/main.js') as f: + c = f.read() +assert '_renderMaturityGauge' in c, 'Missing maturity gauge method' +assert '_renderBarChart' in c, 'Missing bar chart method' +assert \"'gauge-track'\" in c, 'Missing gauge-track class' +assert \"'gauge-segment'\" in c, 'Missing gauge-segment class' +assert \"'gauge-blockers'\" in c, 'Missing gauge-blockers class' +assert \"'paperforge-bar-chart'\" in c, 'Missing bar chart class' +assert \"'bar-fill'\" in c, 'Missing bar-fill class' +assert \"'bar-count'\" in c, 'Missing bar-count class' +assert \"_renderEmptyState\" in c, 'Missing empty state call in bar chart' +assert \"_renderSkeleton\" in c, 'Missing skeleton call in maturity gauge' +print('Task 3 JS verification: ALL CHECKS PASSED') +"</automated> + </verify> + + <done> + _renderMaturityGauge creates 6-segment gauge with level-appropriate filled segments and blocking checks list. _renderBarChart creates horizontal bars proportional to max count with lifecycle stage color classes. Both methods handle null/empty data gracefully. + </done> +</task> + +</tasks> + +<verification> +1. All 7 methods exist on PaperForgeStatusView: `_renderSkeleton`, `_renderEmptyState`, `_buildMetricBar`, `_renderStats` (enhanced), `_renderLifecycleStepper`, `_renderHealthMatrix`, `_renderMaturityGauge`, `_renderBarChart` +2. All methods use `createEl()` DOM API — no `innerHTML` assignments (grep: `innerHTML` only in pre-existing header refresh button and setup wizard) +3. All CSS class names in createEl() cls attributes match the CSS classes defined in Plan 27-01 +4. No new JS files created — all methods on existing PaperForgeStatusView class +5. All rendering methods guard against null/undefined input +6. Skeleton loading and empty state have consistent patterns across all components +7. All unicode characters used for health icons (✓, ⚠, ✗) and checkmark (✓) are valid JavaScript string literals +</verification> + +<success_criteria> +- [ ] All 7 methods added to PaperForgeStatusView class in main.js +- [ ] Enhanced _renderStats() handles null data and adds progress bar for Formal Notes +- [ ] _renderLifecycleStepper renders 6 stages with correct CSS classes (completed/current/pending) +- [ ] _renderHealthMatrix renders 2x2 grid with status icons and tooltip titles +- [ ] _renderMaturityGauge renders 6-segment gauge with level and blockers +- [ ] _renderBarChart renders horizontal proportional bars with lifecycle stage colors +- [ ] All methods gracefully handle null/undefined data +- [ ] CSS class names match Plan 27-01 exactly +- [ ] No innerHTML assignments in new code +- [ ] No new JS files +</success_criteria> + +<output> +After completion, create `.planning/phases/27-component-library/27-02-SUMMARY.md` +</output> diff --git a/.planning/phases/29-per-paper-view/29-01-PLAN.md b/.planning/phases/29-per-paper-view/29-01-PLAN.md new file mode 100644 index 00000000..f5370a51 --- /dev/null +++ b/.planning/phases/29-per-paper-view/29-01-PLAN.md @@ -0,0 +1,542 @@ +--- +phase: 29-per-paper-view +plan: 01 +type: execute +wave: 1 +depends_on: [28-02, 27-02] +files_modified: + - paperforge/plugin/main.js + - paperforge/plugin/styles.css +autonomous: true +requirements: [PAPER-01, PAPER-02, PAPER-03, PAPER-04] + +must_haves: + truths: + - "User opens a paper note with zotero_key — dashboard switches to paper mode with metadata (title, authors, year) at the top" + - "User sees a lifecycle stepper showing the paper's current stage (imported/indexed/pdf_ready/fulltext_ready/deep_read/ai_ready) with completed/current/pending state classes" + - "User sees a 2x2 health matrix with PDF/OCR/Note/Asset cells, color-coded ok/warn/fail with hover tooltips" + - "User sees a maturity gauge (1-6 segments) with level number and blocking checks listed when level < 6" + - "User sees a next-step recommendation card with human-readable action and a clickable trigger button that runs the recommended CLI command or copies the zotero_key for /pf-deep" + - "User sees contextual action buttons above the components: Copy Context (copies JSON to clipboard) and Open Fulltext (opens the fulltext.md file)" + artifacts: + - path: "paperforge/plugin/styles.css" + provides: "Per-paper view layout CSS (Section 15) + next-step card CSS (Section 16)" + contains: ".paperforge-paper-view, .paperforge-paper-header, .paperforge-paper-actions, .paperforge-contextual-btn, .paperforge-next-step-card" + - path: "paperforge/plugin/main.js" + provides: "Replaces _renderPaperMode() placeholder with full per-paper view rendering" + contains: "_renderPaperMode(), _openFulltext()" + key_links: + - from: "_detectAndSwitch()" + to: "_renderPaperMode()" + via: "_switchMode('paper')" + pattern: "_renderPaperMode" + - from: "_renderPaperMode()" + to: "_renderLifecycleStepper()" + via: "this._renderLifecycleStepper(contentContainer, entry, entry.lifecycle)" + pattern: "_renderLifecycleStepper" + - from: "_renderPaperMode()" + to: "_renderHealthMatrix()" + via: "this._renderHealthMatrix(contentContainer, entry.health)" + pattern: "_renderHealthMatrix" + - from: "_renderPaperMode()" + to: "_renderMaturityGauge()" + via: "this._renderMaturityGauge(contentContainer, entry.maturity.level, entry.maturity.blocking)" + pattern: "_renderMaturityGauge" + - from: "_renderPaperMode()" + to: "_runAction()" + via: "next-step action button calls this._runAction()" + pattern: "_runAction" +--- + +<objective> +Wire up the per-paper dashboard view using Phase 27 component render methods, triggered by Phase 28 context detection. Replace the `_renderPaperMode()` placeholder with a full rendering pipeline: metadata header, lifecycle stepper, health matrix, maturity gauge, next-step recommendation card with action trigger, and contextual action buttons (Copy Context, Open Fulltext). + +Purpose: User can see the full lifecycle, health, maturity, and next-step guidance for any individual paper by opening its note and viewing the PaperForge dashboard. +Output: Modified `paperforge/plugin/main.js` + `paperforge/plugin/styles.css` +</objective> + +<execution_context> +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@C:/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md + +@.planning/phases/27-component-library/27-02-SUMMARY.md +@.planning/phases/28-dashboard-shell-context-detection/28-02-SUMMARY.md + +@paperforge/plugin/main.js +@paperforge/plugin/styles.css +@paperforge/worker/asset_state.py +@paperforge/worker/asset_index.py + +<interfaces> +<!-- Key types and contracts needed by the executor --> + +Entry shape from formal-library.json (canonical index entries built in Phase 23 and enriched in Phase 24): +``` +entry = { + "zotero_key": "ABCDEFG", + "title": "Paper Title", + "authors": ["Author1", "Author2"], // array of strings + "year": "2024", + "domain": "骨科", + "lifecycle": "fulltext_ready", // string from compute_lifecycle(): imported|indexed|pdf_ready|fulltext_ready|deep_read_done|ai_context_ready + "health": { + "pdf_health": "ok", // status string or detailed check message + "ocr_health": "done", + "note_health": "healthy", + "asset_health": "healthy" + }, + "maturity": { + "level": 4, // number 1-6 + "blocking": ["Fulltext needed"] // array of strings, empty when level=6 + }, + "next_step": "ocr", // string: sync|ocr|repair|/pf-deep|rebuild index|ready + "fulltext_path": "Literature/domain/key - Title/fulltext.md", + "note_path": "Literature/domain/key - Title.md", +} +``` + +Available render methods (from Phase 27): +```javascript +_renderLifecycleStepper(container, lifecycle, currentStage) + // lifecycle: passed through for truthiness check, currentStage: string matching stages[i].key + // Valid stage keys: imported, indexed, pdf_ready, fulltext_ready, deep_read, ai_ready + +_renderHealthMatrix(container, health) + // health object with keys: pdf_health, ocr_health, note_health, asset_health + // status values: ok/healthy → .ok class, warn/warning → .warn class, anything else → .fail class + +_renderMaturityGauge(container, maturityLevel, blockingChecks) + // maturityLevel: number 1-6, blockingChecks: array of strings + +_renderSkeleton(container) // adds .paperforge-loading class for shimmer +_renderEmptyState(container, message) // shows muted "No data" / custom message +``` + +State references (from Phase 28): +```javascript +this._currentPaperEntry // full entry dict or null (set by _detectAndSwitch → _findEntry) +this._currentPaperKey // zotero_key string or null +this._currentMode // 'global' | 'paper' | 'collection' +this._contentEl // .paperforge-content-area container +``` + +Existing ACTIONS array members relevant to per-paper mode: +- paperforge-copy-context: { id:'paperforge-copy-context', cmd:'context', needsKey:true, ... } +- paperforge-sync: { id:'paperforge-sync', cmd:'sync', ... } +- paperforge-ocr: { id:'paperforge-ocr', cmd:'ocr', ... } +- paperforge-repair: { id:'paperforge-repair', cmd:'repair', ... } + +Existing _runAction(a, card) — a is an action object from ACTIONS, card is the DOM element for status. +</interfaces> +</context> + +<tasks> + +<task type="auto"> + <name>Task 1: Add per-paper view CSS layout and next-step card styles</name> + <files>paperforge/plugin/styles.css</files> + <read_first> + - paperforge/plugin/styles.css (read the last ~80 lines to see Section 13 and 14 patterns and avoid conflicts) + </read_first> + <action> + Append two new CSS sections to `paperforge/plugin/styles.css` AFTER the last Section 14 closing `===` line. Use the exact same section header format as existing sections (e.g., `/* ========================================================================== SECTION N — Name ========================================================================== */`). + + **Section 15 — Per-Paper View:** + + ```css + /* ========================================================================== + SECTION 15 — Per-Paper View Layout + ========================================================================== */ + .paperforge-paper-view { + display: flex; + flex-direction: column; + gap: 20px; + } + + .paperforge-paper-header { + padding-bottom: 8px; + border-bottom: 1px solid var(--background-modifier-border); + } + + .paperforge-paper-title { + font-size: var(--font-ui-large); + font-weight: var(--font-semibold); + color: var(--text-normal); + line-height: 1.4; + margin: 0 0 6px 0; + } + + .paperforge-paper-meta { + font-size: var(--font-ui-small); + color: var(--text-muted); + line-height: 1.5; + } + + .paperforge-paper-authors { + display: inline; + } + + .paperforge-paper-year { + display: inline; + } + + .paperforge-paper-year::before { + content: " \u00B7 "; + } + + .paperforge-paper-actions { + display: flex; + flex-direction: row; + gap: 8px; + flex-wrap: wrap; + } + + .paperforge-contextual-btn { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 6px 14px; + font-size: var(--font-ui-small); + font-weight: var(--font-semibold); + color: var(--text-normal); + background: var(--background-secondary-alt); + border: 1px solid var(--background-modifier-border); + border-radius: var(--radius-m); + cursor: pointer; + transition: background 0.15s, color 0.15s; + user-select: none; + } + + .paperforge-contextual-btn:hover { + background: var(--background-modifier-hover); + color: var(--text-accent); + } + + .paperforge-contextual-btn:active { + transform: scale(0.97); + } + + .paperforge-contextual-btn-icon { + font-size: 14px; + line-height: 1; + } + ``` + + **Section 16 — Next-Step Card:** + + ```css + /* ========================================================================== + SECTION 16 — Next-Step Recommendation Card + ========================================================================== */ + .paperforge-next-step-card { + background: var(--background-secondary); + border: 1px solid var(--background-modifier-border); + border-radius: var(--radius-m); + padding: 16px; + display: flex; + flex-direction: column; + gap: 10px; + } + + .paperforge-next-step-card.ready { + border-color: var(--color-green); + background: color-mix(in srgb, var(--color-green) 8%, var(--background-secondary)); + } + + .paperforge-next-step-label { + font-size: var(--font-ui-smaller); + font-weight: var(--font-semibold); + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-muted); + } + + .paperforge-next-step-text { + font-size: var(--font-ui-small); + color: var(--text-normal); + line-height: 1.5; + } + + .paperforge-next-step-trigger { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 6px 14px; + font-size: var(--font-ui-small); + font-weight: var(--font-semibold); + color: var(--text-on-accent); + background: var(--interactive-accent); + border: none; + border-radius: var(--radius-m); + cursor: pointer; + transition: opacity 0.15s; + align-self: flex-start; + user-select: none; + } + + .paperforge-next-step-trigger:hover { + opacity: 0.85; + } + + .paperforge-next-step-trigger:active { + transform: scale(0.97); + } + + .paperforge-next-step-trigger.running { + opacity: 0.6; + pointer-events: none; + } + ``` + + Insert the above two sections at the end of the file (after the last Section 14 closing fence). Ensure the section numbering is correct — these are Sections 15 and 16, continuing from Phase 28's Section 14. + </action> + <verify> + <automated>rg -n "SECTION 15" paperforge/plugin/styles.css && rg -n "SECTION 16" paperforge/plugin/styles.css</automated> + </verify> + <acceptance_criteria> + - `styles.css` contains `/* SECTION 15 -- Per-Paper View Layout */` + - `styles.css` contains `/* SECTION 16 -- Next-Step Recommendation Card */` + - Class `.paperforge-paper-view` defined in CSS + - Class `.paperforge-next-step-card` defined in CSS + - Class `.paperforge-contextual-btn` defined in CSS + - No syntax errors: `node -e "require('fs').readFileSync('paperforge/plugin/styles.css','utf-8')"` succeeds + </acceptance_criteria> + <done>CSS Sections 15 and 16 added. Per-paper view has layout container, header, meta, actions row, contextual button, and next-step card styling.</done> +</task> + +<task type="auto"> + <name>Task 2: Replace _renderPaperMode() placeholder with full per-paper view — metadata, components, contextual actions, and next-step card</name> + <files>paperforge/plugin/main.js</files> + <read_first> + - paperforge/plugin/main.js lines 813-834 (existing _renderPaperMode placeholder) + - paperforge/plugin/main.js lines 880-992 (existing _runAction method, for reuse pattern) + - paperforge/plugin/main.js lines 557-660 (existing _renderLifecycleStepper, _renderHealthMatrix, _renderMaturityGauge signatures) + - paperforge/plugin/main.js lines 157-208 (ACTIONS array, for matching action objects by cmd) + </read_first> + <action> + Replace the existing `_renderPaperMode()` method (lines 813-834) with the full implementation below. Also add a new helper method `_openFulltext(path)`. + + **Step 1: Replace `_renderPaperMode()`** + + Remove the existing placeholder (lines 813-834). Insert the new implementation: + + ```javascript + /* ── Per-Paper Mode Render (D-01 through D-09, Phase 29) ── */ + _renderPaperMode() { + const entry = this._currentPaperEntry; + const key = this._currentPaperKey; + + // --- Handle loading / empty / not-found states (D-03) --- + if (!key) { + // No key at all (defensive fallback) + this._renderEmptyState(this._contentEl, 'No paper data available.'); + return; + } + + if (!entry) { + // Key exists but entry not found in index (D-18) + this._contentEl.createEl('div', { + cls: 'paperforge-content-placeholder', + text: 'Paper "' + key + '" not found in canonical index. Sync first.', + }); + return; + } + + // --- Create per-paper view container (D-04 through D-09) --- + const view = this._contentEl.createEl('div', { cls: 'paperforge-paper-view' }); + + // ============ Contextual Action Buttons Row (D-10, D-11, D-12, D-13) ============ + const actionsRow = view.createEl('div', { cls: 'paperforge-paper-actions' }); + + // "Copy Context" button (D-11) — reuse existing paperforge-copy-context action + const ctxBtn = actionsRow.createEl('button', { cls: 'paperforge-contextual-btn' }); + ctxBtn.createEl('span', { cls: 'paperforge-contextual-btn-icon', text: '\u2139' }); + ctxBtn.createEl('span', { text: 'Copy Context' }); + ctxBtn.addEventListener('click', () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) this._runAction(action, ctxBtn); + }); + + // "Open Fulltext" button (D-12) — open fulltext.md in Obsidian + if (entry.fulltext_path) { + const ftBtn = actionsRow.createEl('button', { cls: 'paperforge-contextual-btn' }); + ftBtn.createEl('span', { cls: 'paperforge-contextual-btn-icon', text: '\uD83D\uDCC4' }); + ftBtn.createEl('span', { text: 'Open Fulltext' }); + ftBtn.addEventListener('click', () => this._openFulltext(entry.fulltext_path)); + } + + // ============ Paper Metadata Header (D-04) ============ + const header = view.createEl('div', { cls: 'paperforge-paper-header' }); + const titleText = entry.title || 'Untitled'; + header.createEl('div', { cls: 'paperforge-paper-title', text: titleText }); + + const meta = header.createEl('div', { cls: 'paperforge-paper-meta' }); + if (entry.authors && entry.authors.length > 0) { + const authorsStr = entry.authors.join(', '); + meta.createEl('span', { cls: 'paperforge-paper-authors', text: authorsStr }); + } + if (entry.year) { + meta.createEl('span', { cls: 'paperforge-paper-year', text: String(entry.year) }); + } + + // ============ Lifecycle Stepper (D-05) ============ + this._renderLifecycleStepper(view, entry, entry.lifecycle); + + // ============ Health Matrix (D-06) ============ + this._renderHealthMatrix(view, entry.health); + + // ============ Maturity Gauge (D-07) ============ + const maturity = entry.maturity || {}; + this._renderMaturityGauge(view, maturity.level, maturity.blocking); + + // ============ Next-Step Recommendation Card (D-08, D-09) ============ + this._renderNextStepCard(view, entry, key); + } + ``` + + **Step 2: Add `_renderNextStepCard(container, entry, key)` helper** + + Insert this NEW method AFTER `_renderPaperMode()` (after the closing brace of _renderPaperMode): + + ```javascript + /* ── Next-Step Recommendation Card (D-08, D-09) ── */ + _renderNextStepCard(container, entry, key) { + const nextStep = entry.next_step || 'ready'; + + // Map next_step value to human-readable info + const stepInfo = { + 'sync': { label: 'Sync Needed', text: 'This paper needs to be synced from Zotero. Click to run sync.', cmd: 'sync', icon: '\u21BB' }, + 'ocr': { label: 'OCR Needed', text: 'Fulltext is missing but PDF is present. Click to run OCR.', cmd: 'ocr', icon: '\u229E' }, + 'repair': { label: 'Repair Needed', text: 'State divergence or path errors detected. Click to repair.', cmd: 'repair', icon: '\u21BA' }, + 'rebuild index':{ label: 'Rebuild Needed', text: 'Index may be stale. Click to run sync to rebuild.', cmd: 'sync', icon: '\u21BB' }, + '/pf-deep': { label: 'Ready for Deep Reading', text: 'Fulltext is ready. Copy key to use /pf-deep in OpenCode.', cmd: null, icon: '\uD83D\uDD0D' }, + 'ready': { label: 'All Set', text: 'This paper is fully processed and ready for use.', cmd: 'ready', icon: '\u2713' }, + }; + + const info = stepInfo[nextStep] || stepInfo['ready']; + + // Build the card + const card = container.createEl('div', { cls: 'paperforge-next-step-card' }); + if (nextStep === 'ready') card.addClass('ready'); + + card.createEl('div', { cls: 'paperforge-next-step-label', text: 'Recommended Next Step' }); + card.createEl('div', { cls: 'paperforge-next-step-text', text: info.text }); + + if (info.cmd && info.cmd !== 'ready') { + // Action button for sync/ocr/repair/rebuild index — reuse existing _runAction + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: info.icon + ' ' + info.label }); + trigger.addEventListener('click', () => { + const action = ACTIONS.find(a => a.cmd === info.cmd); + if (action) this._runAction(action, trigger); + }); + } else if (nextStep === '/pf-deep') { + // Copy zotero_key to clipboard for /pf-deep (D-09) + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\uD83D\uDCCB Copy Key for /pf-deep' }); + trigger.addEventListener('click', () => { + navigator.clipboard.writeText(key).then(() => { + trigger.setText('\u2713 Copied!'); + new Notice('Zotero key copied: ' + key); + }).catch(() => { + new Notice('[!!] Clipboard write failed', 6000); + }); + }); + } else if (nextStep === 'ready') { + // Show "Copy Context" shortcut for ready state (D-09) + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\u2139 Copy Context' }); + trigger.addEventListener('click', () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) this._runAction(action, trigger); + }); + } + } + ``` + + **Step 3: Add `_openFulltext(fulltextPath)` helper** + + Insert this NEW method after `_renderNextStepCard`: + + ```javascript + /* ── Open Fulltext File in Obsidian (D-12) ── */ + _openFulltext(fulltextPath) { + if (!fulltextPath) { + new Notice('[!!] No fulltext path available for this paper', 6000); + return; + } + // fulltext_path is relative to vault root (e.g., "Literature/domain/key - Title/fulltext.md") + // Use Obsidian API to open the file + const file = this.app.vault.getAbstractFileByPath(fulltextPath); + if (file) { + this.app.workspace.openLinkText(file.path, ''); + } else { + new Notice('[!!] Fulltext file not found: ' + fulltextPath, 6000); + } + } + ``` + + **IMPORTANT IMPLEMENTATION RULES:** + - Insert `_renderNextStepCard` and `_openFulltext` AFTER `_renderPaperMode`, BEFORE `_renderCollectionMode`. Keep all _renderPaperMode-related methods together. + - The `_runAction` method (lines 880-992) expects `ACTIONS.find(a => a.cmd === info.cmd)` to return a valid action object. For `cmd: 'sync'`, it matches ACTIONS[0] (paperforge-sync). For `cmd: 'ocr'`, it matches ACTIONS[1] (paperforge-ocr). For `cmd: 'repair'`, it matches ACTIONS[3] (paperforge-repair). These all exist and need no modification. + - The paperforge-copy-context action with id `'paperforge-copy-context'` already exists in ACTIONS (line 191). It uses `needsKey: true` which reads `zotero_key` from the active file's frontmatter — this works correctly because in per-paper mode the active file IS the paper's `.md` file. + - For `/pf-deep`, there's no CLI command — just copy the key to clipboard. The unicode escape `\uD83D\uDD0D` renders as a magnifying glass icon. + - For the "Open Fulltext" button icon `\uD83D\uDCC4` renders as a document icon. + - All render method calls (lifecycle stepper, health matrix, maturity gauge) pass `view` as the container — they render into the paper view flow. + - Do NOT modify any existing method signatures (especially _runAction, _detectAndSwitch, _switchMode) — these are already working correctly. + - Do NOT modify the existing _renderCollectionMode method. + - `entry.authors` is an array of strings per `_build_entry()` in asset_index.py line 268: `item.get("authors", [])`. + - `entry.year` is a string per line 271: `item.get("year", "")`. + </action> + <verify> + <automated>node -e "const c=require('fs').readFileSync('paperforge/plugin/main.js','utf-8'); const hasPaperMode=c.includes('_renderPaperMode'); const hasNextStep=c.includes('_renderNextStepCard'); const hasFulltext=c.includes('_openFulltext'); console.log('renderPaperMode:',hasPaperMode,'nextStepCard:',hasNextStep,'openFulltext:',hasFulltext); process.exit(hasPaperMode&&hasNextStep&&hasFulltext?0:1)"</automated> + </verify> + <acceptance_criteria> + - `main.js` contains `_renderPaperMode()` method replaced with new implementation + - `main.js` contains `_renderNextStepCard(container, entry, key)` method + - `main.js` contains `_openFulltext(fulltextPath)` method + - `main.js` parses without syntax errors: `node -c paperforge/plugin/main.js` succeeds + - No reference to the old placeholder text "Phase 29 will add" remains in _renderPaperMode + - Next-step mapping covers all 6 states: sync, ocr, repair, rebuild index, /pf-deep, ready + - Action buttons in next-step card for sync/ocr/repair call `_runAction` with correct action from ACTIONS + - Action button for /pf-deep copies `key` to clipboard via `navigator.clipboard.writeText(key)` + - Action button for "ready" state shows "Copy Context" and calls _runAction with paperforge-copy-context + - Contextual action buttons row is created with "Copy Context" and "Open Fulltext" (if entry.fulltext_path exists) + - Paper metadata header renders title, authors (joined by comma), year + - Lifecycle stepper called with `(view, entry, entry.lifecycle)` + - Health matrix called with `(view, entry.health)` + - Maturity gauge called with `(view, maturity.level, maturity.blocking)` where maturity = entry.maturity || {} + </acceptance_criteria> + <done>_renderPaperMode() renders: contextual action buttons (Copy Context, Open Fulltext), paper metadata header (title, authors, year), lifecycle stepper, health matrix, maturity gauge with blocking checks, and next-step recommendation card with appropriate action trigger for each state (sync/ocr/repair/pf-deep/ready). Placeholder text fully replaced.</done> +</task> + +</tasks> + +<verification> +1. Open a paper note (`.md` with `zotero_key` in frontmatter) in Obsidian → PaperForge dashboard shows per-paper view with metadata, components, and actions. +2. Lifecycle stepper highlights current stage with `.current` class, earlier stages show `.completed`, later stages show `.pending`. +3. Health matrix renders 4 color-coded cells (PDF, OCR, Note, Asset) with hover tooltips. +4. Maturity gauge shows 6 segments with filled count matching the paper's level. +5. Next-step card shows the recommended action with explanatory text and a clickable trigger button. +6. "Copy Context" button copies the paper's canonical index entry JSON to clipboard. +7. "Open Fulltext" button (if fulltext_path exists) opens the fulltext.md file in Obsidian. +8. File parses: `node -c paperforge/plugin/main.js` exits 0. +</verification> + +<success_criteria> +- Phase 28 `_switchMode('paper')` → `_renderPaperMode()` renders a complete per-paper dashboard +- All 4 Phase 27 component render methods display with correct data from the canonical index entry +- Next-step recommendation drives user action (CLI command or key copy) +- Contextual actions (Copy Context, Open Fulltext) operate correctly +- No new components created — pure wiring of existing Phase 27 and Phase 28 methods +</success_criteria> + +<output> +After completion, create `.planning/phases/29-per-paper-view/29-01-SUMMARY.md` +</output> diff --git a/.planning/phases/31-bug-fixes/31-01-PLAN.md b/.planning/phases/31-bug-fixes/31-01-PLAN.md new file mode 100644 index 00000000..f47c1a71 --- /dev/null +++ b/.planning/phases/31-bug-fixes/31-01-PLAN.md @@ -0,0 +1,69 @@ +# Plan 31-01: Bug Fixes + +**Phase:** 31 — Bug Fixes +**Goal:** Plugin version number is displayed correctly; meaningless "ai" UI element is removed. +**Depends on:** Nothing +**Requirements:** NAV-02, NAV-03 + +## Tasks + +### Task 1: Add paperforge_version to canonical index envelope + +**File:** `paperforge/worker/asset_index.py` +**Function:** `build_envelope()` +**Change:** Add `paperforge_version` field to the envelope dict. + +```python +from paperforge import __version__ + +def build_envelope(items: list[dict]) -> dict: + return { + "schema_version": CURRENT_SCHEMA_VERSION, + "generated_at": datetime.now(timezone(timedelta(hours=8))).isoformat(), + "paper_count": len(items), + "paperforge_version": __version__, # <-- NEW + "items": items, + } +``` + +**Verification:** +1. Run `python -c "from paperforge.worker.asset_index import build_envelope; e = build_envelope([]); print(e.get('paperforge_version'))"` — must print a valid semver string like `1.8.0`. +2. Run `paperforge sync --index --verbose` or trigger index rebuild — check generated `formal-library.json` contains `"paperforge_version": "1.8.0"`. +3. Plugin dashboard header shows `v1.8.0` (not `v—`). + +### Task 2: Remove "ai" UI element from plugin dashboard + +**Approach:** Code audit to identify the exact element, then surgical removal. + +**Audit plan:** +1. Open plugin in Obsidian with Research_LitControl library (or any vault with papers). +2. Search rendered HTML for any element containing "ai" that appears as a standalone row/line/section. +3. Grep plugin source for likely candidates: + - `grep -n 'ai_ready\|AI Ready\|ai_' paperforge/plugin/main.js` + - `grep -n 'ai_ready\|AI Ready\|ai_' paperforge/plugin/_styles.css` +4. Identify which element the user considers meaningless. +5. Remove the rendering code for that element. + +**Likely candidates:** +- `_renderLifecycleStepper`: `{ key: 'ai_ready', label: 'AI Ready' }` — if this stage always appears empty/no-op, remove it. +- `_renderBarChart`: `{ key: 'ai_ready', label: 'AI Ready', cls: 'stage-ai-ready' }` — same logic. +- Any other rendering artifact. + +**Verification:** +1. Dashboard no longer shows the identified "ai" row in any mode. +2. `git grep -i 'ai.row\|ai_ready'` in `paperforge/plugin/` returns no rendering references (or the references are justified). +3. No JavaScript errors in console. + +## Success Criteria + +1. Plugin header shows actual PaperForge version (e.g., "v1.8.0"). +2. Dashboard excludes the meaningless "ai" row across all render modes. +3. Python→JS version bridge established via envelope (NAV-02 fixed for future phases). +4. Zero regressions in existing dashboard functionality. + +## Rollback + +If either change breaks the dashboard: +```bash +git checkout -- paperforge/worker/asset_index.py paperforge/plugin/main.js paperforge/plugin/_styles.css +``` diff --git a/.planning/phases/31-bug-fixes/31-CONTEXT.md b/.planning/phases/31-bug-fixes/31-CONTEXT.md new file mode 100644 index 00000000..e49eb68a --- /dev/null +++ b/.planning/phases/31-bug-fixes/31-CONTEXT.md @@ -0,0 +1,39 @@ +# Phase 31: Bug Fixes — Context + +## Domain Research + +### Bug 1: Version Number Not Displayed (NAV-02) + +**Root cause (confirmed by code audit):** +- `asset_index.py:build_envelope()` outputs `{schema_version, generated_at, paper_count, items}` — no `paperforge_version` field. +- Plugin `main.js:358` reads version from `_cachedStats.version`, which defaults to `\u2014` (em dash) on first load. +- Plugin `main.js:281` sets initial badge text to `'v\u2014'`. +- `main.js:459` updates badge via `d.version ? 'v' + d.version : 'v\u2014'`. +- `status.py:634` already has `"version": __import__("paperforge").__version__` but as a standalone field in `paperforge status --json` — NOT in the canonical index envelope. +- Since the canonical index is the primary data source the plugin reads, version never propagates. + +**Fix:** +1. Add `"paperforge_version": __import__("paperforge").__version__` to `build_envelope()` output in `asset_index.py:88-93`. +2. Plugin reads it from `_cachedStats.version` automatically (already wired, just data missing). + +### Bug 2: "ai" Row / Element in Dashboard (NAV-03) + +**Preliminary finding:** +- No direct match for "ai" as a dashboard row in plugin JS/CSS. +- The lifecycle stepper has `{ key: 'ai_ready', label: 'AI Ready' }` stage (line 569). +- The bar chart has `{ key: 'ai_ready', label: 'AI Ready', cls: 'stage-ai-ready' }` stage (line 678). +- The health matrix has 4 dimensions: PDF, OCR, Note, Asset — no "ai" dimension. +- The metric cards show: Papers, Formal Notes, Exports — no "ai" metric. + +**Most likely candidates for "ai row":** +- "AI Ready" stage in lifecycle stepper appearing when no data supports it. +- A rendering artifact or stale element from v1.7 development. +- A specific element in the plugin settings or setup wizard. + +**Fix approach:** Audit during execution. Search for any dashboard-rendered "ai" string that appears as a standalone row/section. Remove the rendering code for that element. + +## Constraints + +- **Zero new dependencies** — plugin stays pure Obsidian JS/CSS, Python uses stdlib only. +- **Backward compatible** — adding `paperforge_version` to envelope must not break existing index consumers (it's additive, readers that ignore unknown fields continue working). +- **Thin-shell** — no business logic migration to JS. Version comes from Python `__init__.py`, flows through envelope. diff --git a/.planning/phases/32-deep-reading-mode-detection/32-01-PLAN.md b/.planning/phases/32-deep-reading-mode-detection/32-01-PLAN.md new file mode 100644 index 00000000..34d86e6a --- /dev/null +++ b/.planning/phases/32-deep-reading-mode-detection/32-01-PLAN.md @@ -0,0 +1,161 @@ +# Plan 32-01: Deep-Reading Mode Detection + +**Phase:** 32 — Deep-Reading Mode Detection +**Goal:** Plugin detects `deep-reading.md` files by filename + parent directory pattern and routes to a dedicated `deep-reading` dashboard mode. +**Depends on:** Phase 31 (Bug Fixes) +**Requirements:** DEEP-01 +**CONTEXT:** `.planning/phases/32-deep-reading-mode-detection/32-CONTEXT.md` + +## Tasks + +### Task 1: Extract `_resolveModeForFile()` pure function + +**File:** `paperforge/plugin/main.js` (around line 714) + +Extract a pure function that determines the mode for a given file without side effects: + +```javascript +_resolveModeForFile(file) { + if (!file) return { mode: 'global', filePath: null, key: null }; + + const ext = file.extension; + const filePath = file.path; + + if (ext === 'base') { + return { mode: 'collection', filePath, key: null, domain: file.basename }; + } + + if (ext === 'md') { + // D-01: Check deep-reading.md FIRST (before zotero_key) + if (file.name === 'deep-reading.md') { + const parentDir = file.parent?.name || ''; + // D-02: Parent directory must match {8-char-key} - {Title} pattern + if (/^[A-Z0-9]{8} - .+$/.test(parentDir)) { + const cache = this.app.metadataCache.getFileCache(file); + const key = cache?.frontmatter?.zotero_key; + if (key) { + return { mode: 'deep-reading', filePath, key }; + } + } + } + + // Standard .md handling — check for zotero_key (D-03 fallback) + const cache = this.app.metadataCache.getFileCache(file); + const key = cache?.frontmatter?.zotero_key; + if (key) { + return { mode: 'paper', filePath, key }; + } + return { mode: 'global', filePath, key: null }; + } + + return { mode: 'global', filePath, key: null }; +} +``` + +**Verification:** +- `_resolveModeForFile(null)` → `{ mode: 'global', ... }` +- `_resolveModeForFile(.base file)` → `{ mode: 'collection', ... }` +- `_resolveModeForFile(deep-reading.md in workspace)` → `{ mode: 'deep-reading', key: 'ABCDEFG' }` +- `_resolveModeForFile(deep-reading.md outside workspace)` → `{ mode: 'paper', key: 'ABCDEFG' }` (falls through to zotero_key check) +- `_resolveModeForFile(.md with zotero_key)` → `{ mode: 'paper', key: 'ABCDEFG' }` +- `_resolveModeForFile(.md without zotero_key)` → `{ mode: 'global', ... }` + +### Task 2: Refactor `_detectAndSwitch()` to use pure function + +**File:** `paperforge/plugin/main.js` (around line 714) + +Replace the current `_detectAndSwitch()` method: + +```javascript +_detectAndSwitch() { + const activeFile = this.app.workspace.getActiveFile(); + const resolved = this._resolveModeForFile(activeFile); + + // D-06: Set state vars from resolved mode + this._currentDomain = resolved.domain || null; + this._currentPaperKey = resolved.key || null; + this._currentPaperEntry = resolved.key ? this._findEntry(resolved.key) : null; + + // D-06, D-10: Switch mode with identity guard + this._switchMode(resolved.mode, resolved.filePath); +} +``` + +### Task 3: Update `_switchMode()` with file-path identity guard + +**File:** `paperforge/plugin/main.js` (around line 763) + +```javascript +_switchMode(mode, filePath) { + // D-06: Identity guard — check BOTH mode AND file path + if (this._currentMode === mode && this._currentFilePath === filePath) { + this._refreshCurrentMode(); + return; + } + + this._currentMode = mode; + this._currentFilePath = filePath; + // ... existing content clear and render dispatch ... +} +``` + +Also add `_currentFilePath` initialization: +```javascript +constructor() { + // ... existing init code ... + this._currentMode = null; // 'global' | 'paper' | 'collection' | 'deep-reading' + this._currentFilePath = null; // D-06: for identity guard + // ... rest of existing init ... +} +``` + +### Task 4: Add `deep-reading` case to mode dispatch + +**File:** `paperforge/plugin/main.js` (around line 775, `_switchMode`'s render dispatch) + +```javascript +// Inside _switchMode's mode dispatch section: +case 'deep-reading': + this._renderDeepReadingMode(); + break; +``` + +This creates the dispatch entry point. The actual `_renderDeepReadingMode()` implementation is Phase 33 — for now, it can render a placeholder or empty state. + +```javascript +/* ── Deep-Reading Mode Render (placeholders — Phase 33 populates) ── */ +_renderDeepReadingMode() { + this._contentEl.createEl('div', { + cls: 'paperforge-content-placeholder', + text: 'Deep-reading dashboard — Phase 33 will populate this view.', + }); +} +``` + +### Task 5: Update mode type reference + +**File:** `paperforge/plugin/main.js` + +Add `'deep-reading'` to any mode type checks or render-mode-header logic: + +```javascript +// In _renderModeHeader (around line 1051): +case 'deep-reading': + modeName = 'Deep Reading'; + break; +``` + +## Success Criteria + +1. Opening `deep-reading.md` inside a paper workspace (`Literature/骨科/ABCDEFG - Title/deep-reading.md`) switches dashboard to `deep-reading` mode with placeholder content. +2. `deep-reading.md` with `zotero_key` in frontmatter routes to `deep-reading` mode, NOT per-paper mode (filename check precedes frontmatter check). +3. Switching away from `deep-reading.md` to another file transitions to the correct mode. +4. Identity guard prevents re-render when same file is re-detected (active-leaf-change double-fire). +5. `deep-reading.md` outside a valid workspace parent directory falls through to per-paper mode (if has zotero_key). +6. All existing mode routing still works: .base → collection, .md + zotero_key → paper, no file → global. + +## Rollback + +```bash +git checkout -- paperforge/plugin/main.js +``` diff --git a/.planning/phases/32-deep-reading-mode-detection/32-CONTEXT.md b/.planning/phases/32-deep-reading-mode-detection/32-CONTEXT.md new file mode 100644 index 00000000..3290e995 --- /dev/null +++ b/.planning/phases/32-deep-reading-mode-detection/32-CONTEXT.md @@ -0,0 +1,102 @@ +# Phase 32: Deep-Reading Mode Detection - Context + +**Gathered:** 2026-05-06 +**Status:** Ready for planning + +<domain> +## Phase Boundary + +Plugin detects `deep-reading.md` files (by filename + parent directory pattern) and routes them to a dedicated `deep-reading` dashboard mode, checked BEFORE the existing `zotero_key` frontmatter check. Prevents deep-reading.md from routing to per-paper mode. Phase 33 populates the deep-reading view with content. + +</domain> + +<decisions> +## Implementation Decisions + +### Detection Strategy +- **D-01:** Check filename (`activeFile.basename === 'deep-reading'`) AND parent directory matches `{8-char-key} - {Title}` pattern. +- **D-02:** Insert this check as the FIRST branch inside the `.md` extension handler — BEFORE the `zotero_key` frontmatter check. +- **D-03:** If deep-reading.md is found outside a valid workspace parent pattern, fall back to normal `.md` handling (if has zotero_key → per-paper mode, else → global mode). + +### Key Resolution +- **D-04:** In deep-reading mode, resolve zotero_key from frontmatter (`metadataCache.getFileCache().frontmatter.zotero_key`), same mechanism as per-paper mode. +- **D-05:** Use resolved key to call `_findEntry(key)` to load the paper's canonical index entry. + +### Mode Identity Guard +- **D-06:** Track `_currentFilePath` alongside `_currentMode`. `_switchMode` checks BOTH mode string AND file path before treating as no-op. +- **D-07:** Extract `_resolveModeForFile()` as a pure function that returns `{mode, filePath, key}` given an activeFile. Decouples detection logic from side effects. + +### State Management +- **D-08:** Add `'deep-reading'` to `_currentMode` type: `'global' | 'paper' | 'collection' | 'deep-reading'`. +- **D-09:** Deep-reading mode does NOT modify `_currentDomain` (stays null). Key and entry stored in `_currentPaperKey` / `_currentPaperEntry`. + +### Auto-Refresh +- **D-10:** `_switchMode('deep-reading')` triggers `_renderDeepReadingMode()` similar to how paper mode triggers `_renderPaperMode()`. Content lives in `this._contentEl`. +- **D-11:** Refresh on active-leaf-change (300ms debounce carries from Phase 28), same as other modes. + +### Quick Actions +- **D-12:** Deep-reading mode keeps the same Quick Actions bar as other modes (Sync, OCR, etc.) — consistent with Phase 28 D-07 principle. + +### the agent's Discretion +- Exact regex for parent directory pattern validation +- CSS transition/animation class name for mode switch +- Error handling if `_findEntry()` returns null in deep-reading mode (show placeholder or empty state) + +</decisions> + +<canonical_refs> +## Canonical References + +### Phase scope and requirements +- `.planning/ROADMAP.md` § Phase 32 — Deep-Reading Mode Detection +- `.planning/REQUIREMENTS.md` § DEEP-01 — Mode detection for deep-reading.md (checked BEFORE zotero_key) + +### Prior phase context +- `.planning/phases/28-dashboard-shell-context-detection/28-CONTEXT.md` — Existing `_detectAndSwitch()`, `_switchMode()`, mode identity guard patterns +- `.planning/phases/29-per-paper-view/29-CONTEXT.md` — Per-paper key resolution via frontmatter +- `.planning/phases/31-bug-fixes/31-CONTEXT.md` — Lifecycle stage key alignment (affects deep-reading stage display) + +### Source code +- `paperforge/plugin/main.js` — PaperForgeStatusView class, specifically `_detectAndSwitch()` (lines 714-760), `_switchMode()` (lines 763-780), `_refreshCurrentMode()` (lines 1046+) +</canonical_refs> + +<code_context> +## Existing Code Insights + +### Reusable Assets +- `_detectAndSwitch()` (main.js:714) — Current routing logic, will be extended with deep-reading branch +- `_switchMode()` (main.js:763) — Mode switching with identity check, content clear, render dispatch +- `_findEntry(key)` (main.js:446) — Single paper lookup by zotero_key from canonical index +- `_getCachedIndex()` (main.js:439) — Lazy-loaded, cached canonical index +- `this.app.metadataCache.getFileCache()` — Frontmatter reading (already used for zotero_key) + +### Established Patterns +- `.md + zotero_key in frontmatter` → paper mode (Phase 28 D-03) +- `.base` → collection mode (Phase 28 D-02) +- No active file → global mode (Phase 28 D-04) +- Quick Actions bar visible in all modes (Phase 28 D-07) +- `active-leaf-change` triggers `_detectAndSwitch()` with 300ms debounce (Phase 28 D-08/D-19) + +### Integration Points +- Phase 32 extends `_detectAndSwitch()` — INSERT deep-reading check INSIDE the `if (ext === 'md')` block, BEFORE the zotero_key check +- Phase 33 will consume `_currentMode === 'deep-reading'` in `_switchMode()` dispatch to call `_renderDeepReadingMode()` +- `_currentPaperKey` and `_currentPaperEntry` will be set for Phase 33 consumption +</code_context> + +<specifics> +## Specific Ideas + +- The detection order matters: deep-reading.md must be caught before its zotero_key frontmatter routes to per-paper mode +- Parent directory pattern: `{8-char-alphanumeric} - {anything}` — match `^[A-Z0-9]{8} - .+$` against parent dir name +- Identity guard should prevent mode re-render when same file is re-detected (double-fire from Obsidian) + +</specifics> + +<deferred> +None — discussion stayed within phase scope +</deferred> + +--- + +*Phase: 32-deep-reading-mode-detection* +*Context gathered: 2026-05-06* diff --git a/.planning/phases/32-deep-reading-mode-detection/32-DISCUSSION-LOG.md b/.planning/phases/32-deep-reading-mode-detection/32-DISCUSSION-LOG.md new file mode 100644 index 00000000..ff24ad15 --- /dev/null +++ b/.planning/phases/32-deep-reading-mode-detection/32-DISCUSSION-LOG.md @@ -0,0 +1,58 @@ +# Phase 32: Deep-Reading Mode Detection - Discussion Log + +> **Audit trail only.** Do not use as input to planning, research, or execution agents. +> Decisions are captured in CONTEXT.md — this log preserves the alternatives considered. + +**Date:** 2026-05-06 +**Phase:** 32-deep-reading-mode-detection +**Areas discussed:** Detection boundary, Key resolution, Identity guard, Edge cases + +--- + +## Detection Boundary + +| Option | Description | Selected | +|--------|-------------|----------| +| 仅检测文件名 | 只要文件名是 deep-reading.md 就进入 deep-reading 模式 | | +| 文件名+目录结构 | 文件名是 deep-reading.md 且父目录符合 {key} - {Title} 模式 | ✓ | + +**User's choice:** 文件名+目录结构 — 更精确，避免误触发 + +--- + +## Key Resolution + +| Option | Description | Selected | +|--------|-------------|----------| +| 从文件名解析 | 从父目录名提取 {key} - {Title} 中的 8 位 key | | +| 从 frontmatter | deep-reading.md 的 frontmatter 中读取 zotero_key | ✓ | + +**User's choice:** 从 frontmatter — 与现有 per-paper 模式一致 + +--- + +## Identity Guard + +| Option | Description | Selected | +|--------|-------------|----------| +| 模式+文件路径双重检查 | mode 和文件路径都变了才重建 dashboard | ✓ | +| 仅模式检查 | 沿用当前逻辑只检查 mode 字符串 | | + +**User's choice:** 模式+文件路径双重检查 — 解决 active-leaf-change double-fire 问题 + +--- + +## Edge Cases + +| Option | Description | Selected | +|--------|-------------|----------| +| 显示 global 模式 | 文件名匹配 deep-reading 但目录不符合 → 回退 global | | +| 显示 per-paper 模式 | 回退到 zotero_key 检查 | ✓ | + +**User's choice:** 显示 per-paper 模式 — 如果有 frontmatter key 就按 per-paper 显示 + +--- + +## Deferred Ideas + +None — discussion stayed within phase scope diff --git a/.planning/phases/33-deep-reading-dashboard-rendering/33-01-PLAN.md b/.planning/phases/33-deep-reading-dashboard-rendering/33-01-PLAN.md new file mode 100644 index 00000000..1ceabe3c --- /dev/null +++ b/.planning/phases/33-deep-reading-dashboard-rendering/33-01-PLAN.md @@ -0,0 +1,324 @@ +# Plan 33-01: Deep-Reading Dashboard Rendering + +**Phase:** 33 — Deep-Reading Dashboard Rendering +**Goal:** Render status card, Pass 1 summary card, and AI Q&A history card in deep-reading mode. +**Depends on:** Phase 32 (mode detection provides `_currentPaperEntry` and `_renderDeepReadingMode()` entry point) +**Requirements:** DEEP-02, DEEP-03 +**CONTEXT:** `.planning/phases/33-deep-reading-dashboard-rendering/33-CONTEXT.md` + +## Tasks + +### Task 1: Build deep-reading view container and read data + +Add file reading helpers at the top of `_renderDeepReadingMode()`: + +```javascript +_renderDeepReadingMode() { + const entry = this._currentPaperEntry; + const view = this._contentEl.createEl('div', { cls: 'paperforge-mode-deepreading' }); + + // Read deep-reading.md content + let deepReadingText = ''; + if (entry && entry.deep_reading_path) { + const drFile = this.app.vault.getAbstractFileByPath(entry.deep_reading_path); + if (drFile) { + deepReadingText = await this.app.vault.read(drFile); + } + } + + // Read discussion.json + let discussionData = null; + if (entry && entry.ai_path) { + const aiDir = entry.ai_path.replace(/^\[\[/, '').replace(/\]\]$/, ''); + const djPath = aiDir + '/discussion.json'; + const djFile = this.app.vault.getAbstractFileByPath(djPath); + if (djFile) { + try { + const raw = await this.app.vault.read(djFile); + discussionData = JSON.parse(raw); + } catch (e) { /* file missing or parse error */ } + } + } + + // Render sections + this._renderDeepStatusCard(view, entry); + this._renderDeepPass1Card(view, deepReadingText); + this._renderDeepQACard(view, discussionData); +} +``` + +Note: `_renderDeepReadingMode` needs to become async (or use Obsidian API promises). + +### Task 2: Status card + +```javascript +_renderDeepStatusCard(container, entry) { + const card = container.createEl('div', { cls: 'paperforge-deepreading-card' }); + card.createEl('div', { cls: 'paperforge-deepreading-card-title', text: '📊 状态概览' }); + + if (!entry) { + card.createEl('div', { cls: 'paperforge-deepreading-empty', text: '暂无数据' }); + return; + } + + const statusItems = [ + { label: 'Figure-Map', value: entry.figure_map ? '✅ 已生成' : '⏳ 待生成' }, + { label: 'OCR 状态', value: entry.ocr_status === 'done' ? '✅ 已完成' : '⏳ ' + (entry.ocr_status || '待处理') }, + { label: 'Pass 完成', value: getPassCompletion(entry) }, + { label: '健康状况', value: entry.health && entry.health.pdf_health === 'healthy' ? '✅ 正常' : '⚠️ 需关注' }, + ]; + + for (const item of statusItems) { + const row = card.createEl('div', { cls: 'paperforge-deepreading-status-row' }); + row.createEl('span', { cls: 'paperforge-deepreading-status-label', text: item.label }); + row.createEl('span', { cls: 'paperforge-deepreading-status-value', text: item.value }); + } +} +``` + +Helper for Pass completion: +```javascript +function getPassCompletion(entry) { + const status = entry.deep_reading_status || 'pending'; + if (status === 'done') return '✅ 3/3 (已完成)'; + // Check deep-reading.md for specific pass markers + return '⏳ 待完成'; +} +``` + +### Task 3: Pass 1 summary card + +```javascript +_renderDeepPass1Card(container, text) { + const card = container.createEl('div', { cls: 'paperforge-deepreading-card' }); + card.createEl('div', { cls: 'paperforge-deepreading-card-title', text: '📝 Pass 1 总结' }); + + if (!text || text.trim() === '') { + card.createEl('div', { cls: 'paperforge-deepreading-empty', text: '暂无 Pass 1 总结' }); + return; + } + + // D-04: Extract content after **一句话总览** marker + const extracted = extractPass1Content(text); + if (!extracted) { + card.createEl('div', { cls: 'paperforge-deepreading-empty', text: '暂无 Pass 1 总结' }); + return; + } + + // Render extracted content as markdown-like text + const content = card.createEl('div', { cls: 'paperforge-deepreading-pass1-content' }); + const lines = extracted.split('\n').filter(l => l.trim()); + for (const line of lines) { + if (line.startsWith('### ')) { + content.createEl('h4', { cls: 'paperforge-deepreading-pass1-subheading', text: line.replace('### ', '') }); + } else if (line.startsWith('**') && line.endsWith('**')) { + content.createEl('p', { cls: 'paperforge-deepreading-pass1-marker', text: line }); + } else if (line.trim()) { + content.createEl('p', { cls: 'paperforge-deepreading-pass1-text', text: line }); + } + } +} +``` + +Helper function: +```javascript +function extractPass1Content(text) { + const markers = ['**一句话总览**', '## Pass 1', '**文章摘要**']; + let content = ''; + for (const marker of markers) { + const idx = text.indexOf(marker); + if (idx !== -1) { + // Find the next major break (next ** section heading or end) + const after = idx + marker.length; + const nextMarker = text.indexOf('**证据边界**', after) !== -1 ? text.indexOf('**证据边界**', after) + : text.indexOf('**Figure 导读**', after) !== -1 ? text.indexOf('**Figure 导读**', after) + : text.length; + content = text.substring(after, nextMarker).trim(); + break; + } + } + return content || null; +} +``` + +### Task 4: AI Q&A history card (collapsible sessions, dialog bubbles) + +```javascript +_renderDeepQACard(container, data) { + const card = container.createEl('div', { cls: 'paperforge-deepreading-card' }); + const header = card.createEl('div', { cls: 'paperforge-deepreading-card-header collapsible' }); + header.createEl('div', { cls: 'paperforge-deepreading-card-title', text: '💬 AI 问答记录' }); + + // D-10: Default collapsed + const body = card.createEl('div', { cls: 'paperforge-deepreading-card-body collapsed' }); + + // Toggle on click + header.addEventListener('click', () => { + body.classList.toggle('collapsed'); + header.classList.toggle('expanded'); + }); + + // D-12: Empty states + if (!data || !data.sessions || data.sessions.length === 0) { + body.createEl('div', { cls: 'paperforge-deepreading-empty', text: !data ? '暂无讨论记录' : '暂无问答内容' }); + return; + } + + // D-08: Sessions-based grouping + for (const session of data.sessions) { + const sessionEl = body.createEl('div', { cls: 'paperforge-deepreading-session' }); + const sessionHeader = sessionEl.createEl('div', { cls: 'paperforge-deepreading-session-header' }); + sessionHeader.createEl('span', { text: session.model || 'AI' }); + sessionHeader.createEl('span', { cls: 'paperforge-deepreading-session-date', text: session.started || '' }); + + // D-09: Dialog bubbles + if (session.qa_pairs) { + for (const qa of session.qa_pairs) { + const qBubble = sessionEl.createEl('div', { cls: 'paperforge-deepreading-bubble question' }); + qBubble.createEl('div', { cls: 'bubble-label', text: '问题' }); + qBubble.createEl('div', { cls: 'bubble-text', text: qa.question }); + + const aBubble = sessionEl.createEl('div', { cls: 'paperforge-deepreading-bubble answer' }); + aBubble.createEl('div', { cls: 'bubble-label', text: '解答' }); + aBubble.createEl('div', { cls: 'bubble-text', text: qa.answer }); + } + } + } +} +``` + +### Task 5: CSS additions + +Add to `styles.css` under SECTION 33: + +```css +/* ========================================================================== + SECTION 33 — Deep-Reading Dashboard Mode + ========================================================================== */ +.paperforge-mode-deepreading { + display: flex; + flex-direction: column; + gap: var(--size-4-4, 16px); +} + +.paperforge-deepreading-card { + background: var(--background-primary-alt); + border-radius: var(--radius-m, 8px); + padding: var(--size-4-4, 16px); +} + +.paperforge-deepreading-card-title { + font-weight: 600; + font-size: var(--font-ui-medium); + margin-bottom: var(--size-4-2, 8px); +} + +.paperforge-deepreading-card-header.collapsible { + cursor: pointer; + user-select: none; +} + +.paperforge-deepreading-card-body.collapsed { + display: none; +} + +.paperforge-deepreading-status-row { + display: flex; + justify-content: space-between; + padding: var(--size-4-1, 4px) 0; + border-bottom: 1px solid var(--background-modifier-border); +} + +.paperforge-deepreading-status-label { + color: var(--text-muted); + font-size: var(--font-ui-small); +} + +.paperforge-deepreading-status-value { + font-size: var(--font-ui-small); +} + +.paperforge-deepreading-pass1-content { + line-height: 1.6; +} + +.paperforge-deepreading-pass1-subheading { + margin: var(--size-4-2, 8px) 0 var(--size-4-1, 4px); + color: var(--text-accent); +} + +.paperforge-deepreading-pass1-marker { + font-weight: 600; + color: var(--text-normal); +} + +.paperforge-deepreading-pass1-text { + color: var(--text-muted); + margin: var(--size-4-1, 4px) 0; +} + +/* Dialog bubbles */ +.paperforge-deepreading-session { + margin-bottom: var(--size-4-3, 12px); +} + +.paperforge-deepreading-session-header { + display: flex; + justify-content: space-between; + font-size: var(--font-ui-small); + color: var(--text-faint); + margin-bottom: var(--size-4-2, 8px); +} + +.paperforge-deepreading-bubble { + padding: var(--size-4-2, 8px) var(--size-4-3, 12px); + border-radius: var(--radius-s, 6px); + margin-bottom: var(--size-4-2, 8px); + max-width: 90%; +} + +.paperforge-deepreading-bubble.question { + background: var(--interactive-accent); + color: var(--text-on-accent); + align-self: flex-start; +} + +.paperforge-deepreading-bubble.answer { + background: var(--background-secondary-alt); + color: var(--text-normal); + margin-left: auto; +} + +.bubble-label { + font-size: var(--font-smallest); + opacity: 0.7; + margin-bottom: 2px; +} + +.bubble-text { + font-size: var(--font-ui-small); + line-height: 1.5; +} + +.paperforge-deepreading-empty { + color: var(--text-faint); + font-size: var(--font-ui-small); + padding: var(--size-4-2, 8px); + text-align: center; +} +``` + +## Success Criteria + +1. Opening deep-reading.md shows three cards: status overview, Pass 1 summary, AI Q&A. +2. Status card shows figure-map, OCR, Pass completion, health info from index entry. +3. Pass 1 summary shows `**一句话总览**` content with `###` sub-sections. +4. AI Q&A section is collapsed by default; clicking expands to session groups with dialog bubbles. +5. All 4 empty-state conditions show Chinese fallback text (never JS errors). +6. CSS is scoped under `.paperforge-mode-deepreading`. + +## Rollback + +```bash +git checkout -- paperforge/plugin/main.js paperforge/plugin/styles.css +``` diff --git a/.planning/phases/33-deep-reading-dashboard-rendering/33-CONTEXT.md b/.planning/phases/33-deep-reading-dashboard-rendering/33-CONTEXT.md new file mode 100644 index 00000000..4c25bf48 --- /dev/null +++ b/.planning/phases/33-deep-reading-dashboard-rendering/33-CONTEXT.md @@ -0,0 +1,110 @@ +# Phase 33: Deep-Reading Dashboard Rendering - Context + +**Gathered:** 2026-05-06 +**Status:** Ready for planning + +<domain> +## Phase Boundary + +Render the deep-reading dashboard view when `_currentMode === 'deep-reading'`. Three sections: status card (paper state overview), Pass 1 full-text summary (extracted from deep-reading.md), and AI Q&A history (from discussion.json). All four empty-state conditions render Chinese fallback messages. Phase 32 provides mode routing. + +</domain> + +<decisions> +## Implementation Decisions + +### Status Bar (Information Card) +- **D-01:** Single information card, vertical list style — consistent with health matrix in per-paper view. +- **D-02:** Shows: figure-map status (present/missing), OCR status (done/pending/failed), Pass completion (1/3, 2/3, 3/3), Health status (healthy/warning). +- **D-03:** Reads data from `entry` fields (canonical index) — no additional file reads needed. + +### Pass 1 Summary Extraction +- **D-04:** Marker-based parsing from deep-reading.md content. Priority order: `**一句话总览**` → `**Pass 1**` heading → `**文章摘要**`. First match wins. +- **D-05:** Content after the marker is extracted as the summary text. +- **D-06:** Multiple `###` sub-sections under Pass 1 are included in the rendered card. +- **D-07:** RegExp fallback for formatting variations (bold vs markdown, Chinese vs English markers). + +### AI Q&A Display +- **D-08:** Sessions-based grouping — each discussion session is a collapsible section. +- **D-09:** Q&A pairs displayed as dialog bubbles — question in one color, answer in another. +- **D-10:** AI Q&A section is DEFAULT COLLAPSED; user clicks to expand. +- **D-11:** Reads from discussion.json via vault.adapter.read() (NOT fs.readFileSync — vault-internal file). + +### Empty States +- **D-12:** All four conditions render with Chinese placeholder messages (never JS errors): + - (a) discussion.json missing → "暂无讨论记录" + - (b) empty sessions array → "暂无问答内容" + - (c) missing Pass 1 content → "暂无 Pass 1 总结" + - (d) deep-reading.md not found → "精读文件未找到" + +### Layout +- **D-13:** Three cards stacked vertically: status card, Pass 1 card, AI Q&A card. +- **D-14:** Status card and Pass 1 card expanded by default. AI Q&A card collapsed by default. +- **D-15:** CSS scoped under `.paperforge-mode-deepreading` with `paperforge-deepreading-*` component prefixes. + +### the agent's Discretion +- Exact dialog bubble colors (use existing Obsidian CSS variables for consistency) +- Expanding/collapsing animation timing +- Number of `###` sub-sections shown before truncation (if content is very long) +- Deep-reading.md reading mechanism (vault.adapter.read() or Obsidian API) +</decisions> + +<canonical_refs> +## Canonical References + +### Phase scope and requirements +- `.planning/ROADMAP.md` § Phase 33 — Deep-Reading Dashboard Rendering +- `.planning/REQUIREMENTS.md` § DEEP-02, DEEP-03 + +### Prior phase context +- `.planning/phases/27-component-library/27-CONTEXT.md` — CSS naming conventions, render methods +- `.planning/phases/29-per-paper-view/29-CONTEXT.md` — Card layout patterns +- `.planning/phases/32-deep-reading-mode-detection/32-CONTEXT.md` — Mode routing, _currentPaperEntry, _renderDeepReadingMode() entry point +- `.planning/phases/31-bug-fixes/31-CONTEXT.md` — Lifecycle stage alignment + +### Source code +- `paperforge/plugin/main.js` — _renderDeepReadingMode() placeholder (Phase 32), _renderPaperMode() for layout reference +- `paperforge/plugin/styles.css` — CSS class patterns +</canonical_refs> + +<code_context> +## Existing Code Insights + +### Reusable Assets +- `_renderDeepReadingMode()` (main.js:959) — Empty placeholder from Phase 32, ready to be populated +- `_renderPaperMode()` (main.js:817) — Reference pattern for card layout (createEl, div stacking, CSS classes) +- `_renderHealthMatrix()` (main.js:592) — Reference for information-card-with-icons pattern +- `_getCachedIndex()` — For loading paper entry data +- `_findEntry(key)` — Single paper lookup +- `_currentPaperEntry` — Currently loaded paper entry (set by Phase 32) + +### Established Patterns +- Card layout: `this._contentEl.createEl('div', { cls: 'paperforge-*-view' })` then nested divs +- CSS naming: `.paperforge-*` for components, `paperforge-*-*` for sub-elements +- Color: Obsidian CSS variables (`var(--color-*)`) +- Quick Actions bar remains visible in all modes (Phase 28 D-07) + +### Integration Points +- Called from `_switchMode()` case `'deep-reading'` (Phase 32 D-10) +- `_currentPaperEntry` provides {lifecycle, health, maturity, deep_reading_path, ai_path, ocr_status} +- Phase 34 (Jump to Deep Reading button) depends on Phase 33 rendering being functional +</code_context> + +<specifics> +## Specific Ideas + +- Pass 1 content structure: `**一句话总览**` paragraph followed by multiple `###` sub-sections (研究问题与核心假设, 作者整体研究路线, etc.) +- Status card should feel like a compact overview — not a full health audit, just at-a-glance state +- AI dialog bubbles: question in one color (e.g., `var(--interactive-accent)` tint), answer in another (e.g., `var(--background-primary-alt)`) +- Session collapsible: header shows session date/model, clicking expands to reveal Q&A pairs + +</specifics> + +<deferred> +None — discussion stayed within phase scope +</deferred> + +--- + +*Phase: 33-deep-reading-dashboard-rendering* +*Context gathered: 2026-05-06* diff --git a/.planning/phases/33-deep-reading-dashboard-rendering/33-DISCUSSION-LOG.md b/.planning/phases/33-deep-reading-dashboard-rendering/33-DISCUSSION-LOG.md new file mode 100644 index 00000000..710b4a5d --- /dev/null +++ b/.planning/phases/33-deep-reading-dashboard-rendering/33-DISCUSSION-LOG.md @@ -0,0 +1,71 @@ +# Phase 33: Deep-Reading Dashboard Rendering - Discussion Log + +> **Audit trail only.** Do not use as input to planning, research, or execution agents. +> Decisions are captured in CONTEXT.md — this log preserves the alternatives considered. + +**Date:** 2026-05-06 +**Phase:** 33-deep-reading-dashboard-rendering +**Areas discussed:** Pass 1 extraction, Status bar, AI Q&A display, Layout + +--- + +## Pass 1 Extraction + +| Option | Description | Selected | +|--------|-------------|----------| +| 一级标题解析 | 查找 `## Pass 1` 和 `## Pass 2` 之间内容 | | +| 标记词解析 | 查找 `**一句话总览**` 等标记后的文本 | ✓ | + +**User's choice:** 标记词解析 — 灵活，匹配不同写作风格 + +**Follow-up — Specific markers:** +- User provided sample content showing `**一句话总览**` + paragraph + `###` sub-sections +- Decision: Extract all content from the matched marker onward, including `###` sub-sections +- Priority order: `**一句话总览**` → `**Pass 1**` → `**文章摘要**` + +--- + +## Status Bar + +| Option | Description | Selected | +|--------|-------------|----------| +| 徽章行 | 横向排列徽章 | | +| 信息卡片 | 纵向列表，带图标和文字 | ✓ | + +**User's choice:** 信息卡片 — 与 per-paper health matrix 风格一致 + +--- + +## AI Q&A Display + +| Option | Description | Selected | +|--------|-------------|----------| +| 最近 5 条问答 | 简单列表，最近优先 | | +| 按会话分组 | 每个 session 可展开/折叠 | ✓ | + +**User's choice:** 按会话分组 — 完整展示每次讨论 + +**Follow-up — Q&A format:** +| Option | Description | Selected | +|--------|-------------|----------| +| 问题:... 解答:... | 两行文字格式 | | +| 对话气泡 | 不同颜色背景，类似聊天界面 | ✓ | + +**User's choice:** 对话气泡 — 问题/解答不同颜色 + +--- + +## Layout + +| Option | Description | Selected | +|--------|-------------|----------| +| 纵向堆叠+卡片 | 三个卡片上下排列 | ✓ | +| 可折叠面板 | 全部可折叠 | | + +**User's choice:** 纵向堆叠，AI 问答默认折叠，状态栏和 Pass 1 默认展开 + +--- + +## Deferred Ideas + +None — discussion stayed within phase scope diff --git a/.planning/phases/34-jump-to-deep-reading-button/34-01-PLAN.md b/.planning/phases/34-jump-to-deep-reading-button/34-01-PLAN.md new file mode 100644 index 00000000..3010f3af --- /dev/null +++ b/.planning/phases/34-jump-to-deep-reading-button/34-01-PLAN.md @@ -0,0 +1,242 @@ +--- +phase: 34-jump-to-deep-reading-button +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - paperforge/plugin/main.js +autonomous: true +requirements: + - NAV-01 +must_haves: + truths: + - "Per-paper dashboard card shows jump-to-deep-reading button when deep_reading_path exists AND deep_reading_status is 'done'" + - "Button is fully hidden (not disabled) when conditions are not met — Copy Context shows instead" + - "Clicking the button opens deep-reading.md in the active leaf via openLinkText()" + - "When deep-reading.md is missing from disk despite the index claiming it, a Notice appears instead of a crash" + artifacts: + - path: "paperforge/plugin/main.js" + provides: "LANG i18n keys + modified _renderNextStepCard() ready state" + contains: "jump_to_deep_reading" + key_links: + - from: "_renderNextStepCard() ready state" + to: "entry.deep_reading_path + entry.deep_reading_status" + via: "conditional check before rendering button" + pattern: "deep_reading_status.*done" + - from: "Jump button click handler" + to: "this.app.workspace.openLinkText()" + via: "getAbstractFileByPath() verification first" + pattern: "openLinkText" +--- + +<objective> +Add a "Jump to Deep Reading" button to the per-paper dashboard's next-step recommendation card. + +Purpose: When a paper is fully processed (next_step === 'ready') and has deep_reading_status === 'done', the "All Set" card shows a button that opens the deep-reading.md file in Obsidian — replacing the generic "Copy Context" button. This provides a one-click path from per-paper dashboard into the deep-reading dashboard (Phase 32 → Phase 33 pipeline). + +Output: +- paperforge/plugin/main.js modified with: + - i18n keys in both LANG.zh and LANG.en for button label + - Conditional logic in _renderNextStepCard() ready state to show jump button vs Copy Context +</objective> + +<execution_context> +@C:/Users/Lin/.opencode/get-shit-done/workflows/execute-plan.md +@C:/Users/Lin/.opencode/get-shit-done/templates/summary.md +</execution_context> + +<context> + +@.planning/PROJECT.md +@.planning/STATE.md +@.planning/REQUIREMENTS.md + +@.planning/phases/34-jump-to-deep-reading-button/34-CONTEXT.md + +@paperforge/plugin/main.js + +<interfaces> + +From paperforge/plugin/main.js: + +**LANG object** (lines 17-20): +```js +const LANG = { + en: { /* ~145 keys as single-line object */ }, + zh: { /* ~145 keys as single-line object */ }, +}; +``` +Pattern: each language has a flat key-value object. Keys are snake_case. Values are strings. + +**t() function** (line 145): +```js +function t(key) { return (T && T[key]) || (LANG.en[key]) || key; } +``` +T is set to LANG.zh by default (line 22), then overridden by langFromApp() at runtime. Adding a key to both LANG.en and LANG.zh means t('jump_to_deep_reading') returns the correct locale string. + +**Object.assign(LANG.en) block** (lines 24-81) and **Object.assign(LANG.zh) block** (lines 83-127): +Additional keys appended via Object.assign after the main LANG definition. New keys should follow the same pattern. + +**_renderNextStepCard() ready state** (lines 932-940): +```js +} else if (nextStep === 'ready') { + // Show "Copy Context" shortcut for ready state (D-09) + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\u2139 Copy Context' }); + trigger.addEventListener('click', () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) this._runAction(action, trigger); + }); +} +``` + +**_openFulltext() pattern** (lines 944-957) — reference for jump button click handler: +```js +_openFulltext(fulltextPath) { + if (!fulltextPath) { + new Notice('[!!] No fulltext path available for this paper', 6000); + return; + } + const file = this.app.vault.getAbstractFileByPath(fulltextPath); + if (file) { + this.app.workspace.openLinkText(file.path, ''); + } else { + new Notice('[!!] Fulltext file not found: ' + fulltextPath, 6000); + } +} +``` + +**entry data shape** (consumed from canonical index, established by Phase 32): +``` +entry.deep_reading_path — string, relative vault path to deep-reading.md (e.g., "Literature/Ortho/ABCDEFG - Title/deep-reading.md") +entry.deep_reading_status — string, 'done' | 'pending' | 'processing' | 'failed' +``` + +</interfaces> + +</context> + +<tasks> + +<task type="auto"> + <name>Task 1: Add i18n keys for jump-to-deep-reading button</name> + <files>paperforge/plugin/main.js</files> + <read_first>paperforge/plugin/main.js (lines 17-155 — full LANG object and t() function)</read_first> + <action> + Add a `jump_to_deep_reading` i18n key to both language packs in main.js: + + 1. In the LANG.en initial object (line 18), add the key `jump_to_deep_reading: 'Open Deep Reading',` — insert it before the final closing brace of the en object. + The en object is defined as a single-line object starting at line 18. Insert the new key as the last entry before the closing `}`. + + 2. In the LANG.zh initial object (line 19), add the key `jump_to_deep_reading: '跳转到精读',` — insert it before the final closing brace of the zh object. + The zh object is also a single-line object. Insert as the last entry before the closing `}`. + + Per D-06 and D-07: The label is accessed via `t('jump_to_deep_reading')` at runtime. + No Object.assign blocks need updating — the keys go directly in the LANG.en and LANG.zh literal objects. + </action> + <verify> + <automated>rg -n "jump_to_deep_reading" paperforge/plugin/main.js</automated> + </verify> + <acceptance_criteria> + - `rg "jump_to_deep_reading" paperforge/plugin/main.js` shows exactly 2 matches: 1 in LANG.en ("Open Deep Reading"), 1 in LANG.zh ("跳转到精读") + - The t() function at line 145 remains untouched + - Node.js syntax check: `node -e "require('fs').readFileSync('paperforge/plugin/main.js','utf8').match(/const LANG = \{[\s\S]*?\};/)"` — no parse errors + </acceptance_criteria> + <done>i18n keys added: 'jump_to_deep_reading' resolves to "Open Deep Reading" (en) and "跳转到精读" (zh)</done> +</task> + +<task type="auto"> + <name>Task 2: Modify _renderNextStepCard() ready state to show jump button</name> + <files>paperforge/plugin/main.js</files> + <read_first>paperforge/plugin/main.js (lines 889-957 — _renderNextStepCard function, _openFulltext pattern)</read_first> + <action> + Modify the `ready` case in `_renderNextStepCard()` (currently lines 932-940) to conditionally render either the jump-to-deep-reading button or the existing Copy Context button. + + The current code block (lines 932-940): + ```js + } else if (nextStep === 'ready') { + // Show "Copy Context" shortcut for ready state (D-09) + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\u2139 Copy Context' }); + trigger.addEventListener('click', () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) this._runAction(action, trigger); + }); + } + ``` + + Replace with the following conditional logic (per D-01, D-03, D-04, D-05, D-08): + + ```js + } else if (nextStep === 'ready') { + if (entry.deep_reading_path && entry.deep_reading_status === 'done') { + // D-01, D-03: Show jump-to-deep-reading button when paper has deep reading content + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\uD83D\uDD0D ' + t('jump_to_deep_reading') }); + trigger.addEventListener('click', () => { + // D-04: Follow _openFulltext() pattern — verify file, open, error Notice + const drFile = this.app.vault.getAbstractFileByPath(entry.deep_reading_path); + if (drFile) { + this.app.workspace.openLinkText(drFile.path, ''); + } else { + // D-05: File missing from disk despite index claiming it exists + new Notice('[!!] ' + t('deep_reading_not_found'), 6000); + } + }); + } else { + // D-02, D-03: Fall back to existing Copy Context button + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\u2139 Copy Context' }); + trigger.addEventListener('click', () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) this._runAction(action, trigger); + }); + } + } + ``` + + Also add a second i18n key for the error notice message: + - In LANG.en, add: `deep_reading_not_found: 'Deep reading file not found',` (before the closing brace) + - In LANG.zh, add: `deep_reading_not_found: '精读文件未找到',` (before the closing brace) + + Note: The magnifying glass icon \uD83D\uDD0D is the same unicode used by the `/pf-deep` state at line 899 (per D-08). The `entry` and `t` variables are already in scope — no additional imports needed. + </action> + <verify> + <automated>node -e "const src=require('fs').readFileSync('paperforge/plugin/main.js','utf8'); const hasJumpButton=src.includes('deep_reading_status'); const hasNotFoundKey=src.includes('deep_reading_not_found'); const hasOpenLinkText=src.includes('openLinkText'); console.log('jump_condition:',hasJumpButton,'not_found_key:',hasNotFoundKey,'openLinkText:',hasOpenLinkText)"</automated> + </verify> + <acceptance_criteria> + - `rg "deep_reading_status.*done" paperforge/plugin/main.js` matches in _renderNextStepCard() + - `rg "deep_reading_not_found" paperforge/plugin/main.js` matches exactly 2 times (en + zh) + - `rg "openLinkText" paperforge/plugin/main.js` matches at least 2 times (existing _openFulltext + new jump button handler) + - Node.js test: `node -e "require('fs').readFileSync('paperforge/plugin/main.js','utf8')"` — no syntax errors (JS is valid) + - The old unconditional "Copy Context" render (without the deep_reading_path / deep_reading_status guard) is removed — the else branch keeps Copy Context but only as fallback + </acceptance_criteria> + <done>_renderNextStepCard() ready state conditionally shows jump-to-deep-reading button when deep_reading_path exists and deep_reading_status is 'done', falling back to Copy Context otherwise</done> +</task> + +</tasks> + +<verification> +All checks are automated: + +1. i18n keys present: `rg -c "jump_to_deep_reading" paperforge/plugin/main.js` = 2 +2. i18n keys present: `rg -c "deep_reading_not_found" paperforge/plugin/main.js` = 2 +3. Conditional jump logic: `rg "deep_reading_status" paperforge/plugin/main.js` matches in _renderNextStepCard +4. openLinkText usage: `rg "openLinkText" paperforge/plugin/main.js` >= 2 matches +5. No syntax errors: `node -e "require('fs').readFileSync('paperforge/plugin/main.js','utf8')"` runs without error +6. Verify old unconditional Copy Context is gone: `rg "Copy Context" paperforge/plugin/main.js` — the el branch keeps it, but the unconditional version (without the deep_reading checks) is replaced +</verification> + +<success_criteria> +- [ ] i18n keys added for `jump_to_deep_reading` and `deep_reading_not_found` in both LANG.zh and LANG.en +- [ ] _renderNextStepCard() ready state conditionally checks `entry.deep_reading_path && entry.deep_reading_status === 'done'` +- [ ] Jump button click handler follows _openFulltext() pattern: getAbstractFileByPath → openLinkText → Notice on missing +- [ ] JS syntax valid — no breakage of existing dashboard rendering +- [ ] All 5 locked decisions (D-01, D-03, D-04, D-05, D-08) implemented +- [ ] All 4 roadmap success criteria satisfied +</success_criteria> + +<output> +After completion, create `.planning/phases/34-jump-to-deep-reading-button/34-01-SUMMARY.md` +</output> diff --git a/.planning/phases/34-jump-to-deep-reading-button/34-01-SUMMARY.md b/.planning/phases/34-jump-to-deep-reading-button/34-01-SUMMARY.md new file mode 100644 index 00000000..49f795d1 --- /dev/null +++ b/.planning/phases/34-jump-to-deep-reading-button/34-01-SUMMARY.md @@ -0,0 +1,35 @@ +# Plan 34-01: Jump to Deep Reading Button — Summary + +## What Was Built + +Added a "Jump to Deep Reading" button to the per-paper dashboard's next-step recommendation card. + +### Changes to `paperforge/plugin/main.js` + +**Task 1 — i18n keys:** +- Added `jump_to_deep_reading` to LANG.en ("Open Deep Reading") and LANG.zh ("跳转到精读") +- Added `deep_reading_not_found` to LANG.en ("Deep reading file not found") and LANG.zh ("精读文件未找到") + +**Task 2 — Conditional button rendering in `_renderNextStepCard()`:** +- When `nextStep === 'ready'` AND `entry.deep_reading_path` exists AND `entry.deep_reading_status === 'done'`: shows jump-to-deep-reading button with magnifying glass icon +- Click handler follows `getAbstractFileByPath()` → `openLinkText()` pattern with `Notice()` on missing file +- Falls back to existing "Copy Context" button when deep reading conditions are not met + +### Decisions Implemented (all 5 of 5 from CONTEXT.md) +| Decision | Status | +|----------|--------| +| D-01: Jump button in next-step card (ready state) | ✓ | +| D-03: Button hidden when conditions not met | ✓ | +| D-04: getAbstractFileByPath + openLinkText | ✓ | +| D-05: Notice on missing file | ✓ | +| D-08: Magnifying glass icon | ✓ | + +### Files Modified +- `paperforge/plugin/main.js` — i18n keys + conditional render logic + +## Self-Check +- JS syntax: OK +- i18n keys: 2 keys × 2 languages = 4 entries confirmed +- Conditional condition: `deep_reading_status === 'done'` present in render logic +- `openLinkText`: 2 matches (new button + existing _openFulltext) +- Copy Context preserved as fallback in else branch diff --git a/.planning/phases/34-jump-to-deep-reading-button/34-VERIFICATION.md b/.planning/phases/34-jump-to-deep-reading-button/34-VERIFICATION.md new file mode 100644 index 00000000..31f1badf --- /dev/null +++ b/.planning/phases/34-jump-to-deep-reading-button/34-VERIFICATION.md @@ -0,0 +1,165 @@ +--- +phase: 34-jump-to-deep-reading-button +verified: 2026-05-06T22:00:00Z +status: passed +score: 4/4 must-haves verified +re_verification: false +gaps: [] +human_verification: [] +--- + +# Phase 34: Jump to Deep Reading Button — Verification Report + +**Phase Goal:** Per-paper dashboard card provides a one-click contextual button to open the associated deep-reading.md file, with file-existence verification before navigation. +**Verified:** 2026-05-06T22:00:00Z +**Status:** passed +**Re-verification:** No — initial verification + +--- + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +|---|-------|--------|----------| +| 1 | Per-paper dashboard card shows jump-to-deep-reading button when `deep_reading_path` exists AND `deep_reading_status` is `'done'` | VERIFIED | `main.js:933` — `if (entry.deep_reading_path && entry.deep_reading_status === 'done')` renders button with `t('jump_to_deep_reading')` | +| 2 | Button is fully hidden (not disabled) when conditions are not met — Copy Context shows instead | VERIFIED | `main.js:947-955` — else branch renders Copy Context button as fallback; jump button element is never created | +| 3 | Clicking the button opens deep-reading.md via `openLinkText()` in the active leaf | VERIFIED | `main.js:939-941` — `getAbstractFileByPath()` → `openLinkText(drFile.path, '')` follows the exact same pattern as `_openFulltext()` | +| 4 | When deep-reading.md is missing from disk despite index claiming it exists, a Notice appears instead of crash | VERIFIED | `main.js:942-945` — `if (drFile)` check guards the navigation; else branch fires `new Notice('[!!] ' + t('deep_reading_not_found'), 6000)` | + +**Score:** 4/4 truths verified + +--- + +### Required Artifacts + +| Artifact | Expected | Status | Details | +|----------|----------|--------|---------| +| `paperforge/plugin/main.js` | LANG i18n keys + modified `_renderNextStepCard()` ready state | VERIFIED | Exists: yes. Substantive: `jump_to_deep_reading` + `deep_reading_not_found` in both LANG.en and LANG.zh; conditional jump-button logic at line 933. Wired: integrated into existing `_renderNextStepCard()` flow. | + +--- + +### Key Link Verification + +| From | To | Via | Pattern | Status | Details | +|------|----|-----|---------|--------|---------| +| `_renderNextStepCard()` ready state | `entry.deep_reading_path` + `entry.deep_reading_status` | conditional check before rendering button | `deep_reading_status.*done` | VERIFIED | Match at `main.js:933` — first checks `deep_reading_path` existence, then `deep_reading_status === 'done'` | +| Jump button click handler | `this.app.workspace.openLinkText()` | `getAbstractFileByPath()` verification first | `openLinkText` | VERIFIED | Match at `main.js:941` — `openLinkText(drFile.path, '')` called after `getAbstractFileByPath(entry.deep_reading_path)` | + +--- + +### Data-Flow Trace (Level 4) + +| Artifact | Data Variable | Source | Produces Real Data | Status | +|----------|--------------|--------|-------------------|--------| +| `main.js` jump button conditional | `entry.deep_reading_path`, `entry.deep_reading_status` | `_currentPaperEntry` (from canonical index via `_findEntry()`) | Values originate from `formal-library.json` (Phase 32 index) | FLOWING — button conditionally reads two boolean/scalar fields; not rendering dynamic data, so hollow/stub risk is minimal | + +--- + +### Behavioral Spot-Checks + +| Behavior | Command | Result | Status | +|----------|---------|--------|--------| +| JS syntax valid | `node -e "require('fs').readFileSync('paperforge/plugin/main.js','utf8')"` | No errors | PASS | +| i18n keys defined | `Select-String -Pattern "jump_to_deep_reading" main.js` | 3 matches: 2 definition (en/zh) + 1 usage | PASS | +| Error notice key defined | `Select-String -Pattern "deep_reading_not_found" main.js` | 3 matches: 2 definition (en/zh) + 1 usage | PASS | +| `openLinkText` count | `Select-String -Pattern "openLinkText" main.js` | 2 matches (new jump button + existing `_openFulltext`) | PASS | +| Condition guard present | `Select-String -Pattern "deep_reading_status.*done" main.js` | 1 match at `main.js:933` | PASS | + +--- + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +|-------------|-----------|-------------|--------|----------| +| NAV-01 | 34-01-PLAN.md | Per-paper dashboard card has a "Jump to Deep Reading" button that opens deep-reading.md via `openLinkText()`, verifying file existence with `getAbstractFileByPath()` before navigating | SATISFIED | All 4 success criteria verified. Button renders conditionally, navigates via `openLinkText()`, verifies with `getAbstractFileByPath()`, shows Notice on missing file. Hidden when conditions not met. | + +**Traceability check:** NAV-01 is mapped to Phase 34 in `.planning/REQUIREMENTS.md` traceability table. No orphaned requirements. Requirement fully covered by plan 34-01. + +--- + +### Anti-Patterns Found + +| File | Line | Pattern | Severity | Impact | +|------|------|---------|----------|--------| +| — | — | — | — | None detected in modified code | + +**Scan results:** +- No `TODO`/`FIXME`/`PLACEHOLDER` comments in modified region +- No `console.log` statements in modified region +- No empty implementations (`return null`, `return {}`, `return []`) +- No hardcoded empty data that would block rendering +- Old unconditional "Copy Context" button correctly replaced with conditional logic; Copy Context preserved as else-branch fallback + +--- + +### Human Verification Required + +No items flagged for human verification. All checks are automated and verifiable via static analysis of `main.js`. + +--- + +### Gaps Summary + +**No gaps found.** All 4 observable truths verified, all key links wired, all anti-patterns clear. + +--- + +## Detailed Verification Walkthrough + +### Step 0: Previous Verification Check +No previous VERIFICATION.md found. Initial verification mode. + +### Step 1: Context Loaded +- Phase directory: `.planning/phases/34-jump-to-deep-reading-button/` +- Plan: `34-01-PLAN.md` — single autonomous plan, Wave 1 +- Requirement ID: NAV-01 + +### Step 2: Must-Haves Established +From PLAN frontmatter: +- 4 observable truths +- 1 artifact (`paperforge/plugin/main.js`) +- 2 key links +- 4 roadmap success criteria + +### Step 3-5: Truth, Artifact, and Key Link Verification + +**Truth 1 — Button shows when conditions met:** +Line 933: `if (entry.deep_reading_path && entry.deep_reading_status === 'done')` — both conditions must be true for button to render. This is an exact match with the success criterion. + +**Truth 2 — Button hidden when conditions not met:** +Line 947: `else { // Fall back to existing Copy Context button` — the jump button is only rendered inside the `if` branch. If either condition fails, the button element is never created. This is hiding, not disabling. + +**Truth 3 — openLinkText navigation:** +Lines 939-941: +```js +const drFile = this.app.vault.getAbstractFileByPath(entry.deep_reading_path); +if (drFile) { + this.app.workspace.openLinkText(drFile.path, ''); +} +``` +Follows the exact pattern from `_openFulltext()` (lines 967-972): first verify file existence via `getAbstractFileByPath()`, then `openLinkText()` in the active leaf. + +**Truth 4 — Notice on missing file:** +Lines 942-945: +```js +} else { + new Notice('[!!] ' + t('deep_reading_not_found'), 6000); +} +``` +The i18n key resolves to "Deep reading file not found" (en) or "精读文件未找到" (zh). The 6000ms display duration matches the existing error pattern in `_openFulltext()`. + +### Step 6: Requirements Coverage +NAV-01: All behaviors confirmed. Button renders conditionally, verifies file, navigates, errors gracefully. + +### Step 7: Anti-Patterns +Clean scan. No issues in modified code. + +### Step 9: Overall Status +**passed** — All 4/4 must-haves verified. All key links wired. No gaps. + +--- + +_Verified: 2026-05-06T22:00:00Z_ +_Verifier: the agent (gsd-verifier)_ diff --git a/.planning/research/ARCHITECTURE.md b/.planning/research/ARCHITECTURE.md index 128178df..bd960ef5 100644 --- a/.planning/research/ARCHITECTURE.md +++ b/.planning/research/ARCHITECTURE.md @@ -1,560 +1,371 @@ -# Architecture Patterns +# Architecture Research: v1.8 AI Discussion & Deep-Reading Dashboard -**Domain:** PaperForge v1.6 literature asset foundation integration -**Researched:** 2026-05-03 +**Domain:** Obsidian plugin dashboard + Python CLI hybrid +**Researched:** 2026-05-06 +**Confidence:** HIGH (verified against existing source code at `paperforge/plugin/main.js` and `paperforge/worker/asset_index.py`) -## Recommended Architecture +## System Overview -Evolve the existing `formal-library.json` into the canonical asset index rather than introducing a second competing index. The current code already has a single write point for the file in `paperforge/worker/sync.py` (`run_index_refresh`) and a stable path contract in `_utils.pipeline_paths()["index"]`. Reusing that artifact preserves the current layout, minimizes migration risk, and avoids a new “which index is truth?” problem. - -The target model should be: - -```text -paperforge.json / vault_config -> configuration truth -library-records/*.md -> user intent truth -ocr/<key>/meta.json + fulltext/images/json -> OCR asset truth -Literature/<domain>/<key> - <title>.md -> formal note + deep-reading truth -indexes/formal-library.json -> canonical derived read model -plugin dashboard / status / health / maturity UI -> thin-shell views over canonical index +``` +┌──────────────────────────────────────────────────────────────────┐ +│ Obsidian Plugin (JS/TS) — THIN SHELL │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ PaperForgeStatusView (_detectAndSwitch) │ │ +│ │ Mode Detection: │ │ +│ │ no file → 'global' │ │ +│ │ .base file → 'collection' │ │ +│ │ deep-reading.md → 'deep-reading' ←── NEW v1.8 │ │ +│ │ .md + zotero_key → 'paper' │ │ +│ │ .md other → 'global' │ │ +│ ├────────────────────────────────────────────────────────────┤ │ +│ │ _renderGlobalMode() │ _renderCollectionMode() │ │ +│ │ _renderPaperMode() │ _renderDeepReadingMode() ← NEW │ │ +│ ├────────────────────────────────────────────────────────────┤ │ +│ │ _getCachedIndex() ──reads── formal-library.json │ │ +│ │ _renderDeepReadingMode ──reads── deep-reading.md │ │ +│ │ ──reads── ai/discussion.json │ │ +│ └────────────────────────────────────────────────────────────┘ │ +├──────────────────────────────────────────────────────────────────┤ +│ File System (Obsidian Vault) │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ Literature/{domain}/{key} - {title}/ │ │ +│ │ ├── {key} - {title}.md (formal note) │ │ +│ │ ├── deep-reading.md (extracted from note) │ │ +│ │ ├── fulltext.md (OCR output) │ │ +│ │ ├── figures/ (extracted charts) │ │ +│ │ └── ai/ ←── v1.6 created │ │ +│ │ ├── discussion.json ←── NEW v1.8 JS reads │ │ +│ │ └── discussion.md ←── NEW v1.8 Python writes│ │ +│ └────────────────────────────────────────────────────────────┘ │ +├──────────────────────────────────────────────────────────────────┤ +│ Python CLI (paperforge) │ +│ ┌────────────────────────────────────────────────────────────┐ │ +│ │ paperforge/worker/sync.py — generates deep-reading.md │ │ +│ │ paperforge/worker/asset_index.py — builds formal-library.json│ │ +│ │ paperforge/skills/literature-qa/ld_deep.py — /pf-deep │ │ +│ │ paperforge/commands/context.py — AI context packs │ │ +│ │ [NEW] AI discussion recorder — writes ai/discussion.* │ │ +│ └────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ ``` -This keeps PaperForge local-first and file-based, but makes one important shift: lifecycle, health, readiness, and maturity are no longer recomputed ad hoc by each surface. They are derived once in Python and published through the canonical index. +## Component Responsibilities -## Architecture Diagram +| Component | Responsibility | Owner | Modified in v1.8 | +|-----------|----------------|-------|------------------| +| `_detectAndSwitch()` | Active file → mode resolution | JS | **YES** — add `deep-reading.md` detection | +| `_switchMode()` | Dispatch to mode renderer | JS | **YES** — add `case 'deep-reading'` | +| `_renderDeepReadingMode()` | Deep-reading dashboard UI | JS | **NEW** — entirely new method | +| `_renderPaperMode()` | Per-paper dashboard | JS | **YES** — add "Jump to Deep Reading" button | +| `_buildEnvelope()` | Index envelope with version | Python | **YES** — add `paperforge_version` | +| `_fetchStats()` | Global dashboard data | JS | **YES** — read version from envelope | +| AI discussion recorder | Write `discussion.json` + `discussion.md` | Python | **NEW** — agent-level module | +| `ld_deep.py` / `/pf-deep` | Deep reading scaffold + validation | Python | No change | +| `sync.py` `migrate_to_workspace()` | Creates `ai/` directory | Python | No change (already done) | -```text - +----------------------+ - | paperforge.json | - | vault_config | - +----------+-----------+ - | - v - paperforge.config.py - | - +--------------------+--------------------+ - | | | - v v v - selection-sync ocr worker deep-reading sync - (user intent mirror) (asset producer) (agent-result sync) - | | | - +--------------------+--------------------+ - | - v - NEW: asset index builder / state resolver - | - v - <system_dir>/PaperForge/indexes/formal-library.json - | - +---------------+----------------+ - | | - v v - CLI JSON views/status Obsidian plugin dashboard - doctor/repair summaries Base-support mirror fields +## Mode Detection Flow (v1.8) + +``` +_detectAndSwitch() + │ + ├─ activeFile === null ──────────────────────────→ 'global' + │ + ├─ activeFile.extension === 'base' ──────────────→ 'collection' + │ + ├─ activeFile.extension === 'md' + │ │ + │ ├─ basename === 'deep-reading' ←── NEW CHECK (BEFORE zotero_key) + │ │ AND path matches workspace pattern + │ │ │ + │ │ ├─ extract parent key from path → 'deep-reading' + │ │ │ (Literature/{domain}/{key} - {title}/deep-reading.md) + │ │ └─ set _currentDeepReadingKey, _currentDeepReadingEntry + │ │ + │ └─ frontmatter has zotero_key ──────────────→ 'paper' + │ + └─ fallback ────────────────────────────────────→ 'global' ``` -## Decision: Evolve `formal-library.json`, do not add a second canonical index +**Critical ordering**: The `deep-reading.md` check must come BEFORE the generic `.md + zotero_key` check. Why: `deep-reading.md` is extracted from the formal note and may carry the same frontmatter (including `zotero_key`). We need to route it to the deep-reading dashboard, not the per-paper dashboard. -### Recommendation +**Detection strategy**: Check `activeFile.basename === 'deep-reading'` AND verify the parent directory matches workspace naming pattern `{8-char key} - {slug}`. Extract the zotero_key from the parent directory name (first 8 characters before ` - `). -Keep the path: +## deep-reading.md Detection — Integration Details -`<system_dir>/PaperForge/indexes/formal-library.json` +The detection needs to handle the workspace path structure: -but change its schema from a bare list of note rows to a versioned envelope: +``` +Literature/{domain}/{zotero_key} - {title_slug}/deep-reading.md +``` + +```javascript +// Proposed addition to _detectAndSwitch(), inserted before the .md + zotero_key check: +if (ext === 'md' && activeFile.basename === 'deep-reading') { + // Check if parent directory matches workspace pattern + const parentDir = activeFile.parent ? activeFile.parent.name : ''; + const workspaceMatch = parentDir.match(/^([A-Z0-9]{8})\s+-\s+(.+)$/); + if (workspaceMatch) { + const extractedKey = workspaceMatch[1]; + this._currentPaperKey = extractedKey; + this._currentPaperEntry = this._findEntry(extractedKey); + this._currentDomain = null; + this._switchMode('deep-reading'); + return; + } + // Not in a workspace dir — fall through to global + this._switchMode('global'); + return; +} +``` + +## New: _renderDeepReadingMode() Component Design + +### Data Sources +| Source | How Read | Fields Used | +|--------|----------|-------------| +| `formal-library.json` | `_findEntry(key)` via `_getCachedIndex()` | lifecycle, health, ocr_status, deep_reading_status, maturity, title, year, authors | +| `deep-reading.md` | `this.app.vault.getAbstractFileByPath()` → `read()` | Pass 1 summary, full text sections | +| `ai/discussion.json` | `fs.readFileSync()` (same pattern as index) | AI Q&A history, session metadata | + +### Sub-Components +``` +_renderDeepReadingMode() + │ + ├─ [Status Bar] — inline row showing: + │ lifecycle badge, ocr_status badge, deep_reading_status badge, maturity level + │ + ├─ [Paper Info Header] — title, authors, year, domain + │ (reuses existing _renderPaperMode header pattern) + │ + ├─ [Pass 1 Summary Section] — "## 1. 第一遍：概览" content from deep-reading.md + │ Rendered as formatted callout block + │ + ├─ [AI Discussion History] — "## AI 问答记录" section + │ ├─ discussion.json → rendered as Q&A accordion/cards + │ │ Each session: model badge, timestamp, command, message pairs + │ └─ Empty state: "No AI discussions yet. Use /pf-deep or /pf-paper in OpenCode." + │ + └─ [Navigation] — "← Back to Paper" link + Opens the formal note: this.app.workspace.openLinkText(entry.main_note_path, '') +``` + +### discussion.json Schema (Python → JS contract) ```json { - "schema_version": "2", - "generated_at": "2026-05-03T12:00:00Z", - "config_snapshot": { - "system_dir": "99_System", - "resources_dir": "03_Resources", - "literature_dir": "Literature", - "control_dir": "LiteratureControl", - "base_dir": "05_Bases" - }, - "summary": { - "total_assets": 0, - "health": {}, - "maturity": {} - }, - "items": [ + "format_version": "1", + "zotero_key": "ABCDEFGH", + "sessions": [ { - "zotero_key": "ABCDEFG", - "domain": "骨科", - "title": "...", - "intent": { "analyze": false, "do_ocr": false }, - "assets": { "pdf": {}, "ocr": {}, "note": {}, "figures": {} }, - "state": { "lifecycle": "ocr_ready", "ai_ready": false }, - "health": { "library": "healthy", "pdf": "ok", "ocr": "pending" }, - "maturity": { "score": 35, "level": 2, "next_step": "run_ocr" }, - "paths": { "record": "...", "note": "...", "ocr_meta": "..." } + "session_id": "2026-05-06T143022", + "timestamp": "2026-05-06T14:30:22+08:00", + "model": "deepseek-v4-pro", + "command": "/pf-deep", + "summary": "三阶段精读：生物力学分析", + "message_count": 12, + "messages": [ + { + "role": "user", + "content": "/pf-deep ABCDEFGH", + "timestamp": "2026-05-06T14:30:22+08:00" + }, + { + "role": "assistant", + "content": "## Pass 1: 概览\n\n...", + "timestamp": "2026-05-06T14:31:05+08:00" + } + ] } ] } ``` -### Why not a new file? +**Design rationale**: Sessions array allows append-only writes. The `summary` field provides a one-line title for the dashboard card. `message_count` enables quick size display without parsing all messages. Individual messages are kept (not just summary) for potential future expand/collapse UI. -| Option | Verdict | Why | -|---|---|---| -| Add `asset-index.json` beside `formal-library.json` | Reject | Duplicates responsibility and creates migration drift immediately | -| Rename `formal-library.json` to a new filename | Reject for v1.6 | Touches path contracts and upgrade logic without enough payoff | -| Evolve `formal-library.json` in place | Accept | Lowest-risk change, existing path already centralized | +## Python vs JS Ownership Boundaries -### Migration rule +| Concern | Python Owns | JS Owns | Rationale | +|---------|-------------|---------|-----------| +| Lifecycle computation | ALL (`compute_lifecycle`) | Reads result | Business logic stays in Python | +| Health computation | ALL (`compute_health`) | Reads result | Business logic stays in Python | +| Maturity computation | ALL (`compute_maturity`) | Reads result | Business logic stays in Python | +| Next-step recommendation | ALL (`compute_next_step`) | Reads result | Business logic stays in Python | +| `discussion.json` writing | ALL (agent commands) | Reads only | Agent interaction produces the data | +| `discussion.md` writing | ALL (agent commands) | Reads only | Human-readable companion to JSON | +| Deep-reading content | ALL (`ld_deep.py`, `sync.py`) | Reads via vault API | Parsing/building is Python domain | +| Deep-reading dashboard rendering | None | ALL (CSS + DOM) | Plugin renders, never computes | +| "Jump to" button logic | None | ALL (openLinkText) | Obsidian API operation | +| Version number | Writer (`build_envelope`) | Reader (`_fetchStats`) | Source of truth: `paperforge/__init__.py` | -Add one tolerant reader in Python that accepts both: -- legacy list format -- new envelope format with `items` +**Thin-shell rule preserved**: The JS plugin NEVER computes lifecycle, health, maturity, or next-step. It only reads pre-computed values from `formal-library.json` or directly from workspace files. All business logic remains in Python. -Writers should emit only the new envelope after v1.6 migration. +## Key Data Flow: AI Discussion Recording -## Data Ownership Boundaries - -### 1. Configuration truth - -**Authoritative source:** `paperforge.json` + `vault_config` - -**Owner:** `paperforge/config.py` - -**Rule:** plugin settings are not runtime truth. They are only a UI cache/draft until written back through Python. - -Implication: -- `paperforge/plugin/main.js` should stop owning independent defaults like `System`, `Resources`, `Notes`, `Index_Cards`, `Base`. -- plugin should load resolved config from Python or from `paperforge.json`, then persist only as a mirror. -- any config-changing UI should write via setup/config command flow, not invent JS-only state. - -### 2. User intent truth - -**Authoritative source:** `library-records/<domain>/<key>.md` frontmatter - -**User-owned fields:** -- `analyze` -- `do_ocr` -- future manual workflow flags - -**Machine-mirrored fields:** -- `has_pdf` -- `pdf_path` -- `ocr_status` -- `deep_reading_status` -- `fulltext_md_path` -- future `asset_state`, `library_health`, `maturity_score`, `next_step` - -Rule: users may edit intent fields; workers may overwrite mirrored fields. - -### 3. OCR asset truth - -**Authoritative source:** -- `ocr/<key>/meta.json` -- `ocr/<key>/fulltext.md` -- `ocr/<key>/json/result.json` -- `ocr/<key>/images/*` - -**Owner:** `paperforge/worker/ocr.py` - -Rule: no other component should invent OCR completion state without validating these files. - -### 4. Deep-reading truth - -**Authoritative source:** formal note content under `## 🔍 精读` - -**Owner:** agent output + `has_deep_reading_content()` / `run_deep_reading()` - -Rule: `deep_reading_status` is derived from actual note content, not from the plugin. - -### 5. Canonical derived truth - -**Authoritative source for lifecycle/health/maturity/dashboard:** `formal-library.json` - -**Owner:** new asset-index builder in Python - -Rule: plugin, `status`, health UI, and future context pack features read from this index instead of recomputing from filesystem independently. - -## Recommended New/Modified Artifacts - -### New Python modules - -| Artifact | Type | Responsibility | -|---|---|---| -| `paperforge/worker/asset_index.py` | New | Read all source artifacts, derive canonical item rows, write `formal-library.json` | -| `paperforge/worker/asset_state.py` | New | Pure lifecycle/readiness/health/maturity derivation functions | -| `paperforge/worker/context_pack.py` | New | Build per-paper/per-collection AI context packs from canonical items | -| `paperforge/commands/config.py` | New | JSON get/set surface for plugin config sync | -| `paperforge/commands/context_pack.py` | New | CLI wrapper for ask-this-paper / ask-this-collection / copy-context-pack | - -### Modified Python modules - -| Artifact | Change | -|---|---| -| `paperforge/config.py` | Make resolved config the single runtime truth; expose stable JSON-serializable config payload | -| `paperforge/worker/sync.py` | Split note-writing from index-building; call asset index refresh after selection/index changes | -| `paperforge/worker/ocr.py` | After each touched key, refresh canonical index rows for those keys | -| `paperforge/worker/deep_reading.py` | Refresh canonical index after syncing deep-reading status | -| `paperforge/worker/status.py` | Read summary counts from canonical index instead of recounting raw files where possible | -| `paperforge/worker/repair.py` | Repair source artifacts first, then refresh canonical index | -| `paperforge/worker/base_views.py` | Add derived display columns/filters fed by mirrored fields from canonical index | -| `paperforge/setup_wizard.py` | Stop writing duplicated top-level path fields unless needed for backward compatibility; prefer `vault_config` | -| `paperforge/plugin/main.js` | Read config/dashboard data from Python, never own lifecycle logic | - -### Modified on-disk artifacts - -| Artifact | Status | Notes | -|---|---|---| -| `<system_dir>/PaperForge/indexes/formal-library.json` | Evolve | Canonical index envelope | -| `library-records/*.md` | Extend | Add derived mirror fields for Base/UI filtering | -| `paperforge.json` | Tighten | Single config truth; preserve legacy top-level keys only as compatibility shim | - -### New on-disk artifacts - -| Artifact | Purpose | -|---|---| -| `<system_dir>/PaperForge/indexes/context-packs/<key>.md` | Cached per-paper AI context pack | -| `<system_dir>/PaperForge/indexes/context-packs/collections/<slug>.md` | Cached per-collection AI context pack | -| `<system_dir>/PaperForge/indexes/health-report.json` | Optional summarized health snapshot for dashboard and diagnostics | - -## State Model - -The new model should explicitly separate intent, assets, and derived state. - -### Intent layer - -From `library-record` frontmatter: - -```text -analyze -do_ocr +``` +User runs /pf-deep <key> in OpenCode Agent + ↓ +ld_deep.py prepares scaffold, generates deep-reading content + ↓ +Agent conversation complete + ↓ +[NEW: AI Discussion Recorder module] (Python) + ├─ Captures conversation messages + ├─ Writes ai/discussion.md (human-readable markdown log) + └─ Writes ai/discussion.json (structured JSON) + ↓ +User opens deep-reading.md in Obsidian + ↓ +_detectAndSwitch() detects deep-reading.md → 'deep-reading' mode + ↓ +_renderDeepReadingMode() reads: + ├─ formal-library.json → lifecycle/health/maturity badges + ├─ deep-reading.md → Pass 1 summary text + └─ ai/discussion.json → AI Q&A history cards ``` -This is what the user wants. +## "Jump to Deep Reading" Button — Integration Point -### Asset layer +**Location**: `_renderPaperMode()`, after the next-step card, as an additional contextual action. -Observed from files: +**Condition**: Show when `entry.deep_reading_path` is non-empty AND `entry.deep_reading_status === 'done'`. -```text -export present? -pdf resolvable? -ocr meta present? -ocr payload complete? -formal note present? -deep-reading content present? -figure assets present? -``` - -This is what actually exists. - -### Derived state layer - -Computed in Python only: - -```text -imported -indexed -pdf_ready -ocr_requested -ocr_processing -fulltext_ready -figure_ready -deep_read_ready -deep_read_done -ai_context_ready -``` - -Recommended item schema section: - -```json -"state": { - "lifecycle": "fulltext_ready", - "readiness": { - "pdf": true, - "ocr": true, - "deep_read": false, - "ai_context": false - }, - "next_actions": ["generate_context_pack", "run_pf_deep"] +```javascript +// In _renderPaperMode(), after _renderNextStepCard(): +if (entry.deep_reading_path && entry.deep_reading_status === 'done') { + const drBtn = view.createEl('button', { cls: 'paperforge-contextual-btn deep-reading-btn' }); + drBtn.createEl('span', { cls: 'paperforge-contextual-btn-icon', text: '\uD83D\uDD0D' }); + drBtn.createEl('span', { text: 'Jump to Deep Reading' }); + drBtn.addEventListener('click', () => { + this.app.workspace.openLinkText(entry.deep_reading_path, ''); + }); } ``` -## Read/Write Flow +## Bug Fix Analysis -### Worker write flow +### 1. Version Number Not Displaying -```text -Zotero export change - -> run_selection_sync() - -> update library-records (intent + mirrors) - -> refresh_asset_index(keys=changed) +**Root cause**: `_fetchStats()` at line 358 uses `version: this._cachedStats?.version || '\u2014'` but the `formal-library.json` envelope (from `build_envelope()`) has no `version` field — only `schema_version`, `generated_at`, `paper_count`, `items`. -OCR job change - -> run_ocr() - -> update meta.json/fulltext/assets - -> refresh_asset_index(keys=touched) +**Fix strategy**: +- **Python** (`asset_index.py`): Add `paperforge_version` to `build_envelope()`: + ```python + from paperforge import __version__ + return { + "schema_version": CURRENT_SCHEMA_VERSION, + "paperforge_version": __version__, + "generated_at": ..., + "paper_count": len(items), + "items": items, + } + ``` +- **JS** (`main.js`): Read from envelope in `_fetchStats()`: + ```javascript + version: index.paperforge_version || index.version || '\u2014' + ``` -Deep-reading state change - -> run_deep_reading() - -> sync deep_reading_status mirror - -> refresh_asset_index(keys=touched) +### 2. Meaningless "ai" Row in Dashboard -Repair change - -> run_repair() - -> fix source artifacts - -> refresh_asset_index(keys=fixed) -``` +**Root cause**: Unknown — likely a leftover from an earlier Phase 25-26 UI experiment. Need to inspect what creates this row. Based on the state report (`STATE.md` line 61: 'Dashboard bug: "ai" row is meaningless'), this is a rendering artifact in the global dashboard view. -### Plugin read/write flow +**Fix strategy**: Audit `_renderGlobalMode()` and `_fetchStats()` for any rendering that produces an "ai" label or row. The canonical index entry has `ai_path` as a path string, and the lifecycle includes `ai_context_ready`. If any UI component renders raw field names, filter it out or map to a human label. -```text -Plugin action button - -> spawn `python -m paperforge <command>` - -> command mutates source artifacts - -> command refreshes canonical index - -> plugin re-reads JSON summary/query output +## Architectural Patterns Used -Plugin dashboard load - -> spawn `python -m paperforge status --json` for summary - -> spawn `python -m paperforge context-pack/index query --json` for lists/detail - -> render only; no lifecycle recomputation in JS -``` +### Pattern 1: Mode Dispatch (existing, extended) -### Critical rule +**What**: `_detectAndSwitch()` → `_switchMode()` → `case 'mode': renderX()` +**When**: Adding any new dashboard context +**v1.8 extension**: Add `case 'deep-reading': this._renderDeepReadingMode()` to the switch +**Trade-offs**: Linear dispatch is simple but the switch statement grows. Acceptable for 4 modes. -The plugin should never determine: -- whether OCR is truly complete -- whether a paper is AI-ready -- whether health is red/yellow/green -- what the next step should be +### Pattern 2: File-based Index as Bridge -Those are Python-derived fields. +**What**: `formal-library.json` is the contract between Python and JS. Python writes derived fields (lifecycle, health, maturity, next_step). JS reads and renders. +**When**: Any time JS needs to display computed state +**v1.8 use**: Deep-reading dashboard reads lifecycle/health/maturity from index, only reads `discussion.json` directly for AI-specific data. +**Trade-offs**: Adds a file dependency but eliminates cross-runtime communication complexity. -## Integration Points in Existing Code +### Pattern 3: ReadFileSync Data Loading -### `paperforge/config.py` - -Current strength: already centralizes precedence. Current problem: plugin defaults drift from Python defaults and setup writes both top-level keys and `vault_config`. - -Recommendation: -- keep `load_vault_config()` as the only resolver -- add a JSON export helper used by plugin -- prefer nested `vault_config` as canonical -- keep top-level path keys only as compatibility read support, not required write support - -### `paperforge/worker/sync.py` - -Current role is overloaded: -1. ingest export rows -2. write library-records -3. write formal notes -4. write `formal-library.json` - -Recommendation: -- leave `run_selection_sync()` focused on control records -- leave `run_index_refresh()` focused on formal notes -- move canonical index assembly into `asset_index.py` -- have both call `refresh_asset_index()` rather than hand-building `index_rows` - -This is the cleanest way to integrate new lifecycle/health fields without making `sync.py` larger. - -### `paperforge/worker/ocr.py` - -Current role is already the source of OCR truth via `meta.json` and validation. Keep it that way. - -Recommendation: -- do not push health logic into the plugin -- after status transitions, refresh canonical rows for touched keys -- expose validated OCR health in index as derived fields, not ad hoc status strings scattered across surfaces - -### `paperforge/worker/deep_reading.py` - -Current role is status synchronization, which fits v1.6 well. - -Recommendation: -- continue deriving deep-read completion from note content -- publish the result into canonical index and mirrored record fields -- let maturity scoring depend on this derived state - -### `paperforge/worker/status.py` - -Current role manually counts files and scans frontmatter. - -Recommendation: -- keep doctor-style filesystem checks in place -- move dashboard/status summaries to canonical index summary where possible -- use raw scans only as fallback if index missing/corrupt - -This gives one consistent count across CLI and plugin. - -### `paperforge/worker/repair.py` - -Current role already handles state divergence. - -Recommendation: -- extend it to compare source artifacts vs canonical index freshness -- never repair the index directly; repair sources, then rebuild index - -### `paperforge/worker/base_views.py` - -Current Base strategy is still good. Do not replace Bases with a new system. - -Recommendation: -- add columns like `asset_state`, `library_health`, `maturity_level`, `next_step` -- continue filtering on `analyze`, `do_ocr`, `ocr_status`, `deep_reading_status` -- derive these extra display fields from canonical index and mirror them back into library-record frontmatter during refresh - -This preserves current user workflow while making the dashboard and Bases agree. - -### `paperforge/plugin/main.js` - -Current plugin is correctly thin in command execution, but too independent in configuration and too limited in dashboard data. - -Recommendation: -- keep CommonJS and the current shell approach; do not do a front-end rewrite -- add a config fetch path from Python -- add dashboard JSON fetches backed by canonical index -- keep all writes as CLI subprocesses -- keep settings as UI cache only, not truth - -## Patterns to Follow - -### Pattern 1: Canonical read model over file truths -**What:** derive one authoritative JSON read model from multiple file-backed truths. -**When:** whenever multiple surfaces need the same counts, health, lifecycle, or readiness logic. -**Example:** - -```python -item = build_asset_item( - export_row=export_row, - record=parse_library_record(record_path), - ocr_meta=load_meta(meta_path), - note_state=inspect_note(note_path), -) -``` - -### Pattern 2: User-intent fields stay editable, derived fields stay overwriteable -**What:** keep manual controls and machine mirrors separate in the same markdown record. -**When:** preserving current Base workflow without losing canonical derivation. -**Example:** - -```yaml -analyze: true # user-owned -do_ocr: true # user-owned -asset_state: "ocr_ready" # machine-owned mirror -library_health: "warning" # machine-owned mirror -maturity_score: 55 # machine-owned mirror -``` - -### Pattern 3: Thin-shell plugin, thick Python semantics -**What:** JS renders and triggers commands; Python decides meaning. -**When:** any dashboard, health, maturity, or AI-ready feature. -**Example:** - -```text -plugin -> python -m paperforge status --json -> render cards -plugin -> python -m paperforge context-pack PAPER123 --json -> render/copy -``` +**What**: JS reads JSON files with `fs.readFileSync(indexPath, 'utf-8')` + caching via `_cachedItems` +**When**: All dashboard data loading +**v1.8 extension**: Same pattern for `discussion.json`: `fs.readFileSync(discussionPath, 'utf-8')` from `ai/` directory. +**Trade-offs**: Requires vault-relative path resolution. OK because `this.app.vault.adapter.basePath` provides vault root. ## Anti-Patterns to Avoid -### Anti-Pattern 1: Second lifecycle implementation in the plugin -**What:** JS infers OCR/deep-read/health from file presence or cached settings. -**Why bad:** guaranteed drift from CLI and workers. -**Instead:** plugin reads Python-produced JSON only. +### Anti-Pattern 1: Business Logic in JS -### Anti-Pattern 2: Treating `formal-library.json` as user-editable data -**What:** manual edits or plugin patch writes into canonical index. -**Why bad:** index becomes a source instead of a read model. -**Instead:** source artifacts change first; index is regenerated. +**What people do**: Re-implement lifecycle or health computation in JavaScript +**Why it's wrong**: Creates configuration drift between Python and JS, violates thin-shell constraint +**Do this instead**: Only read pre-computed values from the canonical index. The JS plugin is a view layer only. -### Anti-Pattern 3: Making `library-record` the source of all truth -**What:** stuffing every derived health/lifecycle decision into frontmatter and reading only that. -**Why bad:** repeats current divergence problems. -**Instead:** frontmatter stores intent plus mirrors; source truths remain separate. +### Anti-Pattern 2: Two Frontmatter Sources of Truth -### Anti-Pattern 4: Creating a greenfield asset database layer -**What:** SQLite/daemon/service rewrite for v1.6. -**Why bad:** violates local-first simplicity and existing file-based contract. -**Instead:** improve the current JSON + markdown architecture. +**What people do**: Read deep-reading status from frontmatter AND from the index, compare them +**Why it's wrong**: Creates reconciliation bugs. The index IS the canonical source. +**Do this instead**: Deep-reading dashboard gets all status fields from `_findEntry(key)` which reads from `_getCachedIndex()`. -## Scalability Considerations +### Anti-Pattern 3: Filename-based heuristic for mode detection -| Concern | At 100 users/papers | At 10K papers | At 1M papers | -|---|---|---|---| -| Index rebuild cost | Full rebuild acceptable | Prefer keyed/incremental refresh | Would need sharding or DB, out of scope | -| Plugin dashboard load | Read whole index fine | Add filtered JSON queries and summary endpoints | Full file read impractical | -| Base mirrors | Direct frontmatter update fine | Batch writes should be keyed and minimal | Too many markdown rewrites | -| Context packs | On-demand generation fine | Cache per paper/collection | Need streaming/chunking | +**What people do**: Check if filename === "deep-reading.md" without verifying workspace context +**Why it's wrong**: A user could have any file named "deep-reading.md" anywhere in their vault +**Do this instead**: Verify parent directory matches workspace pattern `{8-char key} - {title_slug}` and the file is inside `Literature/{domain}/` -For v1.6, optimize for incremental per-key refresh, not a new database. +## Recommended Build Order -## Suggested Build Order +Based on dependency analysis, the suggested build order for v1.8 phases: -### Phase 1: Configuration truth hardening -- Normalize plugin defaults to Python defaults -- Add config JSON read surface -- Make `vault_config` the canonical write target -- Verify plugin, CLI, setup, and workers resolve identical paths +1. **Phase 31a: Fix version number display** — 1 change Python (build_envelope), 1 change JS (_fetchStats). Low risk, no dependencies. -### Phase 2: Canonical index builder -- Add `asset_index.py` -- Evolve `formal-library.json` schema -- Add tolerant legacy reader -- Make `run_index_refresh()` delegate index writing here +2. **Phase 31b: Fix "ai" row bug** — Audit and remove. Depends on understanding current rendering, which we already have. -### Phase 3: Unified state + health derivation -- Add lifecycle/readiness/health/maturity pure functions in `asset_state.py` -- Feed from library-records, OCR meta, formal notes -- Mirror selected display fields back into library-record frontmatter +3. **Phase 32: Add deep-reading mode detection** — Extend `_detectAndSwitch()` and `_switchMode()`. Pure JS change. No dependency on Python. -### Phase 4: Status/repair/dashboard convergence -- Make `status --json` read canonical summary -- Make `repair` rebuild index after fixes -- Update plugin dashboard to read canonical summaries/query endpoints +4. **Phase 33: Build _renderDeepReadingMode() component** — New render method with status bar, Pass 1 summary, and placeholder for AI history. Depends on Phase 32. -### Phase 5: Maturity scoring + next-step recommendations -- Add scoring rules to canonical items -- Surface next-step guidance in plugin and Base views +5. **Phase 34: Add "Jump to Deep Reading" button** — Modify `_renderPaperMode()`. Depends on Phase 33 (need consistent deep-reading path resolution). -### Phase 6: AI context packs -- Generate per-paper and per-collection context packs from canonical items -- Store pack paths/readiness in index -- Keep pack bodies outside the main index to avoid bloat +6. **Phase 35: AI discussion recorder (Python)** — Write `discussion.md` + `discussion.json` in `ai/`. Depends on nothing (standalone Python module). -## Practical Field Additions +7. **Phase 36: Wire AI Q&A history into deep-reading dashboard** — Read `discussion.json` in `_renderDeepReadingMode()`. Depends on Phase 33 and Phase 35. -Recommended mirrored `library-record` fields for Bases/UI: +**Rationale**: Fixes first (low risk, quick wins), then detection infrastructure, then rendering, finally data pipeline. -```yaml -asset_state: "fulltext_ready" -pdf_health: "ok" -ocr_health: "warning" -template_health: "ok" -library_health: "warning" -maturity_score: 62 -maturity_level: 3 -next_step: "run_pf_deep" -context_pack_ready: false -``` +## Scaling Considerations -Recommended canonical index sections per item: +| Scale | Architecture Note | +|-------|-------------------| +| ~10 papers | All modes fast. Direct file reads adequate. | +| ~100 papers | Index scan for `_findEntry()` is O(n). Acceptable for this scale. | +| ~1000 papers | `_findEntry()` is still linear scan of items array. Consider Map-based lookup if performance becomes issue. | +| ~10K papers | `_getCachedIndex()` loads full JSON into memory (potentially 10+ MB). Consider paginated index or lazy loading. | -```json -"intent": {}, -"assets": {}, -"state": {}, -"health": {}, -"maturity": {}, -"recommendations": {}, -"paths": {} -``` +*For v1.8: Current scale is ~100 papers. Existing linear scan is fine. No premature optimization needed.* + +## Integration Points Summary + +| Point | From | To | Data | Trigger | +|-------|------|----|------|---------| +| Index reading | `formal-library.json` | `_getCachedIndex()` | All entry fields | On mode switch, on modify event | +| Deep-reading content | `deep-reading.md` | `_renderDeepReadingMode()` | Markdown text | On deep-reading mode entry | +| AI discussion history | `ai/discussion.json` | `_renderDeepReadingMode()` | Session array | On deep-reading mode entry | +| AI discussion writing | Agent command (/pf-deep, /pf-paper) | `ai/discussion.*` | Full conversation | After agent interaction complete | +| Version number | `build_envelope()` | `_fetchStats()` | `paperforge_version` string | On index rebuild | +| Jump to deep-reading | `_renderPaperMode()` button | `app.workspace.openLinkText()` | `deep_reading_path` | User click | ## Sources -- Internal code inspection: `paperforge/config.py` — config precedence and path resolution -- Internal code inspection: `paperforge/worker/_utils.py` — existing `formal-library.json` path contract -- Internal code inspection: `paperforge/worker/sync.py` — current library-record/note/index write flow -- Internal code inspection: `paperforge/worker/ocr.py` — OCR truth and validation flow -- Internal code inspection: `paperforge/worker/deep_reading.py` — deep-reading state sync -- Internal code inspection: `paperforge/worker/status.py` — current summary/dashboard JSON producer -- Internal code inspection: `paperforge/worker/repair.py` — divergence repair model -- Internal code inspection: `paperforge/plugin/main.js` — current plugin thin-shell pattern and config drift risk +- **Verified against source**: `paperforge/plugin/main.js` (lines 1-2067) — existing mode detection, switch, and rendering +- **Verified against source**: `paperforge/worker/asset_index.py` (lines 1-577) — `_build_entry()`, `build_envelope()`, `read_index()` +- **Verified against source**: `paperforge/worker/asset_state.py` (lines 1-243) — lifecycle, health, maturity, next-step computation +- **Verified against source**: `paperforge/worker/sync.py` (lines 1677-1749) — `migrate_to_workspace()`, ai/ directory creation +- **Project context**: `.planning/PROJECT.md` — v1.8 requirements and thin-shell constraint +- **Project state**: `.planning/STATE.md` — bug reports (ai row, version number) + +--- + +*Architecture research for: PaperForge v1.8 AI Discussion & Deep-Reading Dashboard* +*Researched: 2026-05-06* +*Confidence: HIGH — all integration points verified against existing source code* diff --git a/.planning/research/FEATURES.md b/.planning/research/FEATURES.md index e097d0d6..fffaa3c9 100644 --- a/.planning/research/FEATURES.md +++ b/.planning/research/FEATURES.md @@ -1,143 +1,461 @@ -# Feature Landscape +# Feature Research: v1.8 AI Discussion Recording & Deep-Reading Dashboard -**Domain:** AI-ready literature asset management on top of Obsidian + Zotero -**Researched:** 2026-05-03 +**Domain:** AI discussion recording and deep-reading dashboard for Obsidian literature asset management +**Researched:** 2026-05-06 **Confidence:** HIGH -## Framing +## Feature Landscape -This milestone should treat PaperForge as a durable literature asset platform, not a bundle of one-off reading prompts. In this ecosystem, users already expect Zotero to remain the bibliographic source of truth, Obsidian to remain the knowledge workspace, and any AI layer to be grounded in traceable local assets rather than opaque summaries. +### Table Stakes (Users Expect These) -For v1.6, the highest-value features are the ones that make the library legible, repairable, standardized, and reusable by both humans and AI. The strongest differentiators are not "more extraction buttons"; they are reliable lifecycle state, health visibility, and reusable context packaging. +Features users assume exist. Missing these = product feels incomplete. -## Table Stakes +| Category | Feature | Why Expected | Complexity | Notes | +|----------|---------|--------------|------------|-------| +| AI Discussion | **Discussion history in workspace** | Researchers expect to find past AI conversations about a paper next to the paper itself, not scattered in browser tabs or chat logs | MEDIUM | Already have `ai/` directory; just need to populate it | +| AI Discussion | **Human-readable Q&A format** | Users need to browse and reference past discussions without parsing JSON or opening special tools | LOW | Markdown Q&A format (问题:/解答:) is naturally scannable | +| AI Discussion | **Timestamped chronology** | Every AI chat platform shows timestamps; users expect to know "when did I discuss this?" | LOW | Simple date/time stamp per Q&A pair | +| Deep-Reading Dashboard | **Know what deep-reading exists** | When viewing a paper's `deep-reading.md`, users expect to see at a glance whether Pass 1/2/3 are complete, without scrolling through the entire note | MEDIUM | Status bar at the top of the dashboard; parse Pass headers | +| Deep-Reading Dashboard | **Quick navigation to deep-reading content** | The deep-reading.md is a long file; users need a way to jump directly to it from the paper dashboard | LOW | "Jump to Deep Reading" contextual button | +| Navigation | **Jump-to-deep-reading from per-paper card** | If the user is looking at a paper's health/lifecycle dashboard, the natural next question is "can I read the deep analysis?" | LOW | Single button on the per-paper dashboard card | +| Bug Fixes | **Version number shown** | Users expect to confirm which version of PaperForge they're running, especially when reporting issues | LOW | Restore the `_versionBadge` update path | +| Bug Fixes | **No meaningless UI rows** | If a UI element shows irrelevant data (like the "ai" row), it erodes trust in the rest of the dashboard | LOW | Remove or replace with meaningful data | -Features users will expect from the new milestone direction. Missing these makes the product feel like a collection of scripts rather than a literature asset manager. +### Differentiators (Competitive Advantage) -| Category | Feature | Why Expected | Complexity | Existing Dependencies | Notes | -|---------|---------|--------------|------------|------------------------|-------| -| Lifecycle management | **Canonical asset index** | Long-lived libraries need one place to answer: what exists, what is missing, what is usable | High | `paperforge sync`, library-records, formal notes, OCR `meta.json`, `paperforge.json` | This is the foundation. Dashboard and AI entry points should read this, not recompute state separately. | -| Lifecycle management | **Explicit lifecycle state model** | Researchers need stable states like imported, indexed, OCR-ready, fulltext-ready, deep-read, AI-ready | Medium | Existing queue fields, OCR status, deep-reading status | Separate user intent from machine facts and derived readiness. | -| Health diagnostics | **Library health surfaces** | Large libraries inevitably drift; users expect to see broken PDFs, OCR failures, path drift, template drift | Medium | `doctor`, `repair`, path normalization, OCR pipeline | Should be visible per paper and aggregated per collection/library. | -| Asset standardization | **Stable per-asset schema** | AI workflows only scale when metadata, links, fulltext, and notes are predictably shaped | Medium | library-record frontmatter, formal note frontmatter, path normalization | Standardize identifiers, asset paths, provenance fields, and readiness flags. | -| Workflow progression | **Queue and next-step views driven by derived state** | Users expect actionable progression, not raw status fields | Medium | Existing Base views, canonical index, lifecycle model | "What should I do next?" is a table-stakes question for a maintained library. | -| Workflow progression | **Idempotent rebuild / refresh behavior** | Asset managers must safely recompute views after fixes without corrupting notes | Medium | current workers, repair flows, config single source of truth | Essential for trust in long-term use. | +Features that set PaperForge apart from generic Zotero sync or AI chat tools. -## Differentiators +| Category | Feature | Value Proposition | Complexity | Notes | +|----------|---------|-------------------|------------|-------| +| AI Discussion | **Dual-format output: discussion.md + discussion.json** | Human-readable markdown for browsing AND structured JSON for dashboard consumption. No other literature tool bridges this gap with local-first files. | MEDIUM | JSON feeds the dashboard Q&A history card; markdown is the reference copy | +| AI Discussion | **Chronological session grouping with metadata** | Groups Q&A pairs by session (each `/pf-paper` or `/pf-deep` invocation), with start time, model, and agent type recorded. Enables session-level browsing. | MEDIUM | Tier-of-sessions > questions-within-session structure | +| Deep-Reading Dashboard | **Context-aware mode: deep-reading.md auto-detection** | When user opens `deep-reading.md`, the dashboard switches to a dedicated mode showing Pass status, AI Q&A history, and fulltext summary — NOT the per-paper lifecycle dashboard | MEDIUM | Extends existing mode-switching architecture (already has global/paper/collection) | +| Deep-Reading Dashboard | **Pass status overview with completion indicators** | Shows at-a-glance which of the three Keshav passes are complete, with content snippets. Reduces need to scroll through long deep-reading.md files. | MEDIUM | Parse `### Pass 1: 概览`, `### Pass 2: 精读还原`, `### Pass 3: 深度理解` | +| Deep-Reading Dashboard | **Recent AI Q&A from discussion.json** | The dashboard surfaces the most recent AI discussions about the paper directly in the deep-reading mode view. Bridges the gap between dashboard and chat history. | MEDIUM | Read `discussion.json` from `ai/` directory; show last N Q&A pairs | +| AI Discussion | **Voluntary recording model (never auto-triggered)** | Differentiates from "AI logs everything" approaches. Users opt in by running `/pf-paper` or `/pf-deep` — the recording is explicit, not surveillance. | LOW | Recording happens as a side-effect of user-initiated agent commands | -These are the features that can make PaperForge notably better than a generic Zotero-to-Obsidian sync. +### Anti-Features -| Category | Feature | Value Proposition | Complexity | Existing Dependencies | Notes | -|---------|---------|-------------------|------------|------------------------|-------| -| AI context entry points | **Ask-this-paper context pack** | Turns one paper into a reusable, grounded AI entry point with metadata, fulltext, figures, note links, and provenance | Medium | canonical index, OCR fulltext, figure-map, formal note, PDF link | Strong differentiator because it packages existing assets instead of inventing new extraction schemas. | -| AI context entry points | **Ask-this-collection / copy-context-pack** | Lets users move from single-paper workflows to collection-scale synthesis while staying source-grounded | Medium | collection metadata from Zotero, canonical index, note links | Aligns with NotebookLM-style source-grounded workflows, but stays local-first and reusable. | -| Workflow progression | **Library maturity / workflow level scoring** | Gives researchers a simple, motivating signal: how usable is this paper or collection for AI and downstream writing? | Medium | lifecycle model, health diagnostics, canonical index | Valuable if transparent and rule-based; bad if gamified or opaque. | -| Health diagnostics | **Actionable diagnostics with fix paths** | Better than just saying "broken"; tells the user exactly whether to sync, OCR, repair paths, or regenerate a note | Medium | `doctor`, `repair`, queue logic | This converts diagnostics from reporting into workflow guidance. | -| Asset standardization | **Traceable provenance bundle** | Every AI-facing output can point back to PDF, OCR text, notes, and source identifiers | Medium | normalized paths, formal notes, OCR artifacts | Important differentiator for research trust and reuse. | -| Lifecycle management | **Thin-shell plugin dashboard over canonical index** | Keeps UX polished without creating a second business-logic implementation | High | plugin shell, CLI, canonical index, `paperforge.json` | This is product-quality differentiation, not just engineering cleanliness. | +Features that seem good but create problems. -## Anti-Features - -These are attractive distractions for this milestone. Avoid them. - -| Anti-Feature | Why Avoid | What to Do Instead | -|--------------|-----------|-------------------| -| **Productizing discipline-specific extraction outputs** (PICO tables, mechanism tables, parameter tables, etc.) | High maintenance, domain-specific, and fights the platform direction | Ship reusable context packs and templates that specialized prompts can consume | -| **Replacing Zotero as the primary reference manager** | Zotero already owns collections, tags, duplicates, attachments, and citation workflows well | Treat Zotero metadata and attachment relationships as upstream truth | -| **Auto-running deep-reading agents from workers** | Breaks the worker/agent boundary and creates opaque automation | Keep agents user-invoked; improve readiness and context packaging instead | -| **A second lifecycle engine inside the Obsidian plugin** | Creates config drift and inconsistent state calculations | Plugin should read canonical index produced by Python logic | -| **"AI organizes everything" black-box features** | Researchers need traceability and repairability, not magic states they cannot audit | Use explicit states, rule-based maturity, and provenance-preserving outputs | -| **Feature explosion of per-prompt buttons** | Hard to maintain, quickly becomes UX clutter, and turns platform capabilities into brittle one-offs | Support a few generic entry points: ask-paper, ask-collection, copy-context-pack | -| **Cloud-hosted collaboration / sync for this milestone** | Expands scope far beyond the asset-foundation problem and conflicts with local-first constraints | Keep single-user, local-first architecture | -| **Full literature discovery graph product** (Litmaps/ResearchRabbit clone) | Discovery maps are valuable, but they are adjacent to asset reliability and curation, not the current foundation gap | Integrate with external discovery workflows through clean collections/tags/context packs | +| Feature | Why Requested | Why Problematic | Alternative | +|---------|---------------|-----------------|-------------| +| **Auto-recording all agent conversations** | "I never want to lose a conversation" | Creates noise files for abandoned/toy prompts; violates the worker/agent boundary; may capture sensitive or experimental prompts user wants ephemeral | Record only when user runs `/pf-paper` or `/pf-deep` for a specific paper — voluntary and targeted | +| **Full chat transcription in markdown** | "I want to replay the entire conversation" | Long, unstructured, hard to extract value; duplicates what the agent already stores internally | Store structured Q&A pairs with 问题:/解答: format; only meaningful exchanges, not every tool call and status message | +| **Deep-reading dashboard as a replacement for deep-reading.md** | "I just want the dashboard, not the long note" | The deep-reading.md is the authoritative source; a dashboard summary that diverges creates inconsistency and distrust | Dashboard is an index/summary view INTO the deep-reading.md; it links to the full content, never replaces it | +| **Real-time dashboard updates during deep-reading** | "I want to see progress live" | Deep reading is agent-executed outside the plugin; real-time sync would require polling or WebSocket complexity for marginal value | Dashboard refreshes on active-leaf-change (already built); shows final state after deep reading completes | +| **Discussion search across all papers** | "Find any discussion about 'osteoporosis' across my library" | Full-text search is already handled by Obsidian; rebuilding search in the plugin duplicates functionality poorly | Rely on Obsidian's built-in vault search for cross-paper discussion search | ## Feature Dependencies -```text -Single configuration truth (`paperforge.json`) - -> enables canonical asset index +``` +AI Discussion Recorder (discussion.md + discussion.json) + ├──requires──> Existing ai/ directory (v1.6 workspace migration) + ├──requires──> /pf-paper and /pf-deep agent commands exist + └──feeds──> Deep-Reading Dashboard (AI Q&A History card) -Canonical asset index - -> enables lifecycle state model - -> enables health dashboards - -> enables queue views - -> enables maturity scoring - -> enables AI context packs +Deep-Reading Dashboard Mode + ├──requires──> Existing mode-switching architecture (global/paper/collection) + ├──requires──> deep-reading.md exists in paper workspace + ├──requires──> discussion.json exists for AI Q&A card + └──enhances──> Per-paper Dashboard (provides entry point via Jump button) -Stable per-asset schema - -> enables trustworthy index derivation - -> enables reusable AI entry points +Jump-to-Deep-Reading Button + ├──requires──> Per-paper dashboard exists (v1.7) + ├──requires──> deep_reading_path in canonical index + └──triggers──> Deep-Reading Dashboard mode (navigates user there) -Health diagnostics - -> feeds workflow progression - -> feeds maturity scoring +Bug Fix: Version Number + ├──requiress──> _versionBadge element exists (v1.7) + └──requires──> version field populated in _cachedStats -AI context packs - -> require canonical index - -> require provenance fields - -> benefit from OCR/fulltext/figure readiness +Bug Fix: Meaningless "ai" Row + └──removes──> Stale UI element from pre-workspace era ``` -## Milestone Scoping by Requested Category +### Dependency Notes -### 1. Lifecycle management -- Must include: canonical index, explicit state model, recomputation rules -- Should defer: complicated branching workflows or multi-user orchestration +- **AI Discussion Recorder feeds Deep-Reading Dashboard:** The dashboard's AI Q&A History card reads `discussion.json`. Without the recorder, this card shows an empty state. Both features should be built in the same phase. +- **Deep-Reading Dashboard extends paper mode detection:** The existing `_detectAndSwitch()` already handles `.md` files with `zotero_key` frontmatter. The new mode adds a check: if the active file is named `deep-reading.md` and resides in a paper workspace directory, switch to `deep-reading` mode instead of falling back to `global`. +- **Jump-to-Deep-Reading bridges per-paper and deep-reading dashboards:** This is a one-click navigation affordance on the per-paper card. It depends on the `deep_reading_path` field in the canonical index (already populated by `asset_index.py` v1.6). +- **Bug fixes are independent:** They don't block any new feature and can be shipped independently, but bundling in v1.8 improves perceived quality of the new features. -### 2. Health diagnostics -- Must include: PDF health, OCR health, path health, note/template/base health -- Should defer: speculative ML-based anomaly detection +## MVP Definition -### 3. Asset standardization -- Must include: canonical identifiers, normalized paths, provenance fields, consistent frontmatter contract -- Should defer: discipline-specific metadata taxonomies as built-ins +### Must Have (v1.8 Launch) -### 4. AI context entry points -- Must include: ask-this-paper, ask-this-collection, copy-context-pack -- Should defer: fully embedded chat platform, vector DB, or provider-specific lock-in as milestone-defining work +These define the milestone. Without them, "v1.8 AI Discussion & Deep-Reading Dashboard" is not delivered. -### 5. Workflow progression -- Must include: readiness views, next-step recommendations, maturity score/rules -- Should defer: complex automation that removes user control +- [x] **AI Discussion Recorder (discussion.md + discussion.json):** Python recorder module that writes structured Q&A to `ai/discussion.md` (human-readable) and `ai/discussion.json` (dashboard-consumable) when `/pf-paper` or `/pf-deep` is run. +- [x] **Deep-Reading Dashboard Mode:** Plugin detects `deep-reading.md` as active file, switches to `deep-reading` mode showing Pass status summary + AI Q&A history. +- [x] **Jump-to-Deep-Reading Button:** On per-paper dashboard card, a button that opens the paper's `deep-reading.md` in Obsidian and triggers the deep-reading dashboard mode. +- [x] **Bug Fix: Version Number:** Restore version badge display in the plugin header (reads from canonical index or plugin manifest). +- [x] **Bug Fix: Remove meaningless "ai" row:** Identify and remove the stale "ai" UI row from the dashboard. -### 6. What not to productize -- Do not turn every successful prompt into a first-class feature -- Do not hardcode scholarly extraction schemas into core product logic -- Do not duplicate Zotero or NotebookLM product surfaces +### Should Have (v1.8.x Follow-Up) -## Reusable Platform Capabilities vs One-Off Workflows +Deferrable without breaking the milestone. -### Reusable platform capabilities to build -- Canonical index -- Lifecycle and health derivation -- Provenance-preserving asset schema -- Context pack generation -- Dashboard and queue views -- Rule-based maturity / next-step guidance +- [ ] **Discussion session merging:** If a user runs `/pf-paper` twice for the same paper, append to existing `discussion.md` rather than overwrite. +- [ ] **Session metadata in discussion.json:** Record agent type (pf-paper vs pf-deep), model, and duration for each session. +- [ ] **Pass completion percentage in deep-reading dashboard:** Calculate rough completion (filled headings / total headings) for each Pass. -### One-off scholarly workflows to keep optional -- Medical evidence extraction tables -- Domain-specific summary formats -- Prompt-specific synthesis buttons -- Special-case manuscript scaffolds tied to one discipline +### Future Consideration (v2+) -## MVP Recommendation +- [ ] **Discussion search across library:** A dashboard view or Base showing all AI discussions across all papers. +- [ ] **Discussion export to context pack:** Include relevant discussion.json in the AI context pack for follow-up questions. +- [ ] **Deep-reading maturity integration:** Factor deep-reading completion into the existing maturity gauge/score. -Prioritize: -1. **Canonical asset index + lifecycle state model** -2. **Library health surfaces + actionable next-step guidance** -3. **Generic AI context packs for paper and collection scopes** +## Feature Prioritization Matrix -Defer: **Domain-specific extraction products** — they should consume the platform, not define it. +| Feature | User Value | Implementation Cost | Priority | +|---------|------------|---------------------|----------| +| AI Discussion Recorder (discussion.md) | HIGH — researchers lose conversations in chat logs | MEDIUM — new Python module, template, integration with agent commands | P1 | +| AI Discussion Recorder (discussion.json) | MEDIUM — enables dashboard, but markdown is the primary value | MEDIUM — JSON serialization layer on top of discussion.md data | P1 | +| Deep-Reading Dashboard: Pass status | HIGH — immediate answer to "how complete is my reading?" | MEDIUM — new plugin render function, markdown parsing | P1 | +| Deep-Reading Dashboard: AI Q&A history | MEDIUM — surfaces recorded discussions in dashboard | LOW — reads existing discussion.json, renders cards | P1 | +| Deep-Reading Dashboard: Mode detection | HIGH — entry point for the whole feature | LOW — extends existing _detectAndSwitch() | P1 | +| Jump-to-Deep-Reading button | HIGH — bridges paper dashboard to deep reading | LOW — single contextual button + Obsidian openLinkText() | P1 | +| Bug Fix: Version number | MEDIUM — users notice when absent | LOW — single-line fix in _renderStats / _cachedStats | P1 | +| Bug Fix: Remove "ai" row | LOW — cosmetic | LOW — remove dead code | P1 | + +## Feature Details: AI Discussion Recording + +### Discussion Format + +**discussion.md** (human-readable, one file per paper, chronologically appended): + +```markdown +--- +paper_key: ABCDEFG +paper_title: "Mechanisms of Osteoarthritis..." +created: 2026-05-06T10:30:00Z +updated: 2026-05-06T14:45:00Z +session_count: 2 +--- + +# AI Discussions + +## Session 2026-05-06 10:30 — Quick Summary + +**Agent:** /pf-paper +**Started:** 2026-05-06 10:30:00 + +### 问题: What is the main finding of this paper? + +**解答:** The paper demonstrates that Piezo1 mechanosensitive ion channels mediate... +(Observed in Figure 3: IHC staining shows Piezo1 expression in chondrocytes...) + +**来源:** Pass 1 overview, Figure 3 + +--- + +### 问题: What signaling pathways are involved? + +**解答:** The study identifies Ca²⁺/NFAT and YAP/TAZ as downstream pathways... + +--- + +## Session 2026-05-06 14:00 — Deep Reading Discussion + +**Agent:** /pf-deep +**Started:** 2026-05-06 14:00:00 + +### 问题: Is the sample size adequate for the conclusions drawn? + +**解答:** The study uses n=6 per group which is standard for rodent OA models... +However, the power analysis was not reported — a limitation noted in the Discussion. + +**来源:** Pass 3 analysis, Methods section +``` + +**Key design decisions:** +- **问题:/解答: format:** Natural for Chinese-speaking biomedical researchers. Minimally structured — easy to write, easy to scan. +- **Session grouping:** Each `/pf-paper` or `/pf-deep` invocation creates a new session header. Prevents one giant undifferentiated blob. +- **来源 field:** Traces each answer back to specific Pass or section — maintains PaperForge's provenance principle. +- **Append-only:** New sessions append to the end. Never rewrites old sessions. Preserves history naturally. + +**discussion.json** (structured, dashboard-consumable): + +```json +{ + "paper_key": "ABCDEFG", + "schema_version": "1", + "updated": "2026-05-06T14:45:00Z", + "sessions": [ + { + "session_id": "2026-05-06T10:30:00", + "agent": "pf-paper", + "started": "2026-05-06T10:30:00Z", + "qa_pairs": [ + { + "question": "What is the main finding of this paper?", + "answer": "The paper demonstrates that Piezo1 mechanosensitive ion channels mediate...", + "source": "Pass 1 overview, Figure 3", + "timestamp": "2026-05-06T10:31:15Z" + }, + { + "question": "What signaling pathways are involved?", + "answer": "The study identifies Ca²⁺/NFAT and YAP/TAZ as downstream pathways...", + "source": null, + "timestamp": "2026-05-06T10:33:42Z" + } + ] + }, + { + "session_id": "2026-05-06T14:00:00", + "agent": "pf-deep", + "started": "2026-05-06T14:00:00Z", + "qa_pairs": [ + { + "question": "Is the sample size adequate for the conclusions drawn?", + "answer": "The study uses n=6 per group which is standard...", + "source": "Pass 3 analysis, Methods section", + "timestamp": "2026-05-06T14:05:30Z" + } + ] + } + ] +} +``` + +**Key design decisions:** +- **Minimal schema:** Only what the dashboard needs. Avoids duplicating the full markdown content. +- **session_id = ISO timestamp:** Timestamps are unique enough to serve as IDs without a UUID dependency. +- **qa_pairs array:** Flat, filterable, order-preserving. +- **source field optional:** Not all Q&A has a clear source section; null means "general discussion." + +### Recorder Integration Points + +The recorder lives as a new Python module: `paperforge/worker/discussion_recorder.py` + +**Entry points (two):** + +1. **`/pf-paper` completion:** When the agent finishes a `/pf-paper` session, the orchestrator invokes `discussion_recorder.record_session(key, agent="pf-paper", qa_pairs=[...])` +2. **`/pf-deep` completion:** Same pattern, with `agent="pf-deep"` + +**Key constraint from architecture (PROJECT.md):** +> "No auto-triggering of AI agents allowed — recording must be voluntary/user-initiated." + +The recorder is invoked at the END of a user-initiated agent session, writing output to files. It does NOT trigger agents, watch for new prompts, or run in background. + +**File operations:** +- Read existing `discussion.md` / `discussion.json` if they exist +- Append new session data +- Write updated files back to `ai/` directory +- All operations are idempotent: re-running the same session with the same session_id overwrites that session's entry, not duplicates it + +## Feature Details: Deep-Reading Dashboard Mode + +### Mode Detection + +The plugin's `_detectAndSwitch()` adds a new check BEFORE the existing `.md` → `zotero_key` check: + +```javascript +// Pseudocode for deep-reading mode detection: +if (ext === 'md') { + // Check if this is a deep-reading.md in a paper workspace + const parentDir = activeFile.parent?.name || ''; + const workspaceMatch = parentDir.match(/^([A-Z0-9]+) - /); + const isDeepReading = activeFile.basename === 'deep-reading'; + + if (isDeepReading && workspaceMatch) { + // deep-reading mode + this._currentPaperKey = workspaceMatch[1]; // extract zotero_key + this._switchMode('deep-reading'); + return; + } + + // Existing zotero_key check for per-paper mode + // ... +} +``` + +**Detection hierarchy (in order):** +1. `.base` → collection mode (existing) +2. `deep-reading.md` in `{KEY} - {Title}/` workspace → deep-reading mode (NEW) +3. `.md` with `zotero_key` frontmatter → per-paper mode (existing) +4. Everything else → global mode (existing) + +### Dashboard Layout (Deep-Reading Mode) + +``` +┌─────────────────────────────────────────────┐ +│ [DEEP READING] Paper Title (truncated) │ ← Mode badge + context +├─────────────────────────────────────────────┤ +│ │ +│ 📊 Reading Progress │ +│ ┌─────────────────────────────────────┐ │ +│ │ Pass 1: 概览 ✓ Complete │ │ ← Status bar +│ │ Pass 2: 精读还原 ✓ Complete │ │ +│ │ Pass 3: 深度理解 ○ Pending │ │ +│ └─────────────────────────────────────┘ │ +│ │ +│ 📝 Pass 1 Summary │ +│ ┌─────────────────────────────────────┐ │ +│ │ One-liner from 一句话总览 field │ │ ← Snippet from deep-reading.md +│ │ Category: Original Research │ │ +│ │ Context: Osteoarthritis models... │ │ +│ └─────────────────────────────────────┘ │ +│ │ +│ 💬 Recent AI Discussions (2 sessions) │ +│ ┌─────────────────────────────────────┐ │ +│ │ Q: What is the main finding? │ │ ← From discussion.json +│ │ A: The paper demonstrates... [↗] │ │ (last 3 Q&A pairs) +│ ├─────────────────────────────────────┤ │ +│ │ Q: Is the sample size adequate? │ │ +│ │ A: The study uses n=6... [↗] │ │ +│ └─────────────────────────────────────┘ │ +│ │ +│ 🔗 Quick Links │ +│ [Open Fulltext] [Open Main Note] ... │ ← Contextual buttons +│ │ +└─────────────────────────────────────────────┘ +``` + +### Section Details + +#### 1. Reading Progress (Status Bar) + +**What:** Three-line indicator showing Pass completion status. + +**How:** +- Parse `deep-reading.md` content for section markers: + - Pass 1 complete if `### Pass 1: 概览` section has non-scaffold content (not empty, not just template text) + - Pass 2 complete if `### Pass 2: 精读还原` + `#### Figure-by-Figure 解析` have filled content + - Pass 3 complete if `### Pass 3: 深度理解` has filled content +- Simple heuristic: if section has more than X non-empty lines after the heading, it's "complete" +- Visual: ✓ (green) for complete, ○ (gray) for pending, ⚠ (yellow) for partial + +**Why not just check deep_reading_status:** The canonical index's `deep_reading_status` is binary (done/pending). The dashboard needs granularity: a paper might have Pass 1 complete but Pass 2 in progress. Parsing the file directly avoids needing Python-side updates for this granular state. + +#### 2. Pass 1 Summary + +**What:** Quick overview extracted from the deep-reading.md. + +**How:** +- Extract the `**一句话总览**` content (the first non-empty line after that marker) +- Extract the 5 Cs fields from the Pass 1 section +- Show as a card with max 5 lines, "Read more →" link to full deep-reading.md + +#### 3. AI Q&A History + +**What:** Recent Q&A pairs from discussion.json. + +**How:** +- Read `ai/discussion.json` (same workspace directory) +- Show last 3 Q&A pairs across all sessions (most recent first) +- Each card shows: question (truncated to 80 chars) + answer preview (40 chars) +- Click expands to full answer, or opens discussion.md for full context +- Empty state: "No AI discussions yet. Run /pf-paper or /pf-deep to start." + +#### 4. Quick Links + +**What:** Navigation buttons to related files. + +**How:** +- "Open Fulltext" → opens `fulltext.md` in workspace (reuse existing _openFulltext) +- "Open Main Note" → opens the main paper note in workspace +- "Open Discussion" → opens `ai/discussion.md` (if exists) +- These replace the per-paper mode's contextual buttons since the context is different + +### What the Deep-Reading Dashboard is NOT + +- **NOT a replacement for deep-reading.md:** The full note remains the authoritative source. The dashboard is a navigation/view layer. +- **NOT a re-render of the per-paper dashboard:** The lifecycle stepper, health matrix, and maturity gauge belong to the per-paper card. Deep-reading mode is focused on the reading content itself. +- **NOT a real-time monitor:** No polling for agent progress. Dashboards refresh on active-leaf-change (existing pattern). + +## Feature Details: Jump-to-Deep-Reading Button + +### Placement + +On the per-paper dashboard card, in the contextual actions row (where "Copy Context" and "Open Fulltext" currently live). + +### Behavior + +1. **Visible condition:** Button appears when `entry.deep_reading_path` is non-empty in the canonical index (meaning deep-reading.md exists in the workspace). +2. **Click action:** Opens `deep-reading.md` in Obsidian using `this.app.workspace.openLinkText()`. The dashboard auto-detects the new active file and switches to deep-reading mode. +3. **Visual:** Similar to existing contextual buttons (`paperforge-contextual-btn` class), with an icon + text: "🔬 Deep Reading" (or a Unicode character that renders well). + +### Implementation + +```javascript +// In _renderPaperMode(), after the existing contextual buttons: +if (entry.deep_reading_path) { + const drBtn = actionsRow.createEl('button', { cls: 'paperforge-contextual-btn' }); + drBtn.createEl('span', { cls: 'paperforge-contextual-btn-icon', text: '\uD83D\uDD2C' }); // 🔬 + drBtn.createEl('span', { text: 'Deep Reading' }); + drBtn.addEventListener('click', () => { + this.app.workspace.openLinkText(entry.deep_reading_path, '', false); + }); +} +``` + +## Feature Details: Bug Fixes + +### Bug: Version Number Not Displaying + +**Root cause:** The `_versionBadge` element is created in `_buildPanel()` but only updated in `_renderStats()`. The `_cachedStats.version` field is set to `'\u2014'` (em dash) on first load and never overwritten because the version field isn't populated from the canonical index. + +**Fix:** Two changes: +1. In `_fetchStats()`, after building `this._cachedStats`, add: `this._cachedStats.version = index.schema_version || index.version || '\u2014';` +2. Alternatively, read the version from `paperforge/__init__.py` via Python subprocess, or from `paperforge/plugin/manifest.json` plugin version. The canonical index should carry a version field. + +**Simplest fix:** The plugin manifest already has `"version": "1.4.15"`. Read it at plugin init and supply to the badge: + +```javascript +// In onOpen() or _buildPanel(): +const manifest = this.app.plugins.plugins['paperforge']?.manifest; +if (manifest?.version) { + this._versionBadge.setText('v' + manifest.version); +} +``` + +### Bug: Meaningless "ai" Row + +**Root cause:** From the v1.6 workspace migration era, the per-paper dashboard or global dashboard may have a row/label displaying "ai" — likely a leftover from when `ai/` directory creation was tracked as a health dimension. + +**Fix:** Identify the source. Likely in the `_renderHealthMatrix()` call or `_renderNextStepCard()` where a stale field like `health.ai_health` is being rendered. Remove any reference to an "ai" health dimension from: +1. `paperforge/worker/asset_index.py` — `compute_health()` function +2. `paperforge/plugin/main.js` — `_renderHealthMatrix()` dimensions array +3. `paperforge/plugin/main.js` — `_fetchStats()` health aggregation + +## Competitor / Ecosystem Analysis + +| Tool | Discussion Recording | Dashboard Integration | Notes | +|------|---------------------|-----------------------|-------| +| **Obsidian Smart Chat** | Saves thread URLs + status inline in notes; markdown codeblocks | Dataview dashboards from `chat-active`/`chat-done` fields | Closest pattern: inline recording + dashboard querying. PaperForge differs by having dedicated per-paper workspace + structured Q&A format | +| **Obsidian Copilot** | Saves conversations as `.md` with YAML frontmatter; project-aware isolation | Chat history popover with fuzzy search, time grouping | Strong inspiration for session grouping and naming conventions | +| **Gemini Scribe** | Per-note history file: `[Note] - Gemini History.md`; auto-appending | History files linked to source notes; graph integration | One-file-per-note model — simple, but doesn't support multi-session grouping well | +| **Claude Sessions** | Reads Claude Code JSONL logs; renders interactive timeline | Summary dashboard with hero cards, token charts, tool charts; Base dashboards | Over-engineered for PaperForge's needs (full timeline rendering), but the "summary dashboard + Bases" pattern is relevant | +| **Smart2Brain** | Save chats; continue later | RAG-powered Q&A with vault note references | Focuses on knowledge retrieval, not discussion recording | +| **Omi Conversations** | Auto-sync from Omi device; daily-indexed conversations | Hub dashboard with tabs (tasks, conversations, memories, stats, map) | The folder-per-day + index structure is clean. PaperForge's session-per-invocation is simpler and sufficient | + +**Key insight from ecosystem:** The Obsidian plugin ecosystem overwhelmingly favors **file-based recording** (markdown in the vault) over database-backed storage. PaperForge's `discussion.md` + `discussion.json` dual-format approach fits this pattern while adding dashboard-queryable structure that pure markdown lacks. + +## Architecture Alignment + +All new features must respect existing architecture principles from `.planning/research/ARCHITECTURE.md`: + +1. **Thin-shell plugin:** The plugin reads JSON (canonical index + discussion.json), never recomputes lifecycle or health. The deep-reading mode parsing (Pass status) is a read-only display concern — not business logic duplication. +2. **Python-owned truth:** Discussion recording logic lives in `paperforge/worker/discussion_recorder.py`. The plugin only reads the output files. +3. **No new canonical index fields needed:** The deep-reading dashboard reads `deep_reading.md` content directly and `discussion.json` from the workspace. No changes to `formal-library.json` required (though a `discussion_count` field could be added later for the per-paper card). +4. **Mode switching extends existing pattern:** Deep-reading mode is a fourth mode (`deep-reading`) alongside global, paper, and collection. It uses the same `_switchMode()`, `_renderModeHeader()`, and `_refreshCurrentMode()` infrastructure. ## Sources -- Project direction and scope guardrails: `.planning/PROJECT.md` — HIGH confidence -- Zotero official docs: Collections and Tags (updated 2025-07-29) — https://www.zotero.org/support/collections_and_tags — HIGH confidence -- Zotero official docs: Adding Files to your Zotero Library (updated 2025-09-11) — https://www.zotero.org/support/attaching_files — HIGH confidence -- Zotero official docs: Duplicate Detection — https://www.zotero.org/support/duplicate_detection — MEDIUM confidence (older doc, still aligned with current product model) -- Obsidian official help: Bases / Bases syntax / Properties — via Context7 official help mirror — HIGH confidence -- Google official help: NotebookLM grounded chat with inline citations — https://support.google.com/notebooklm/answer/16164461?hl=en — HIGH confidence -- ResearchRabbit product positioning (discovery/visualization/organization) — https://www.researchrabbit.ai/ — MEDIUM confidence -- Litmaps product positioning (discover/visualize/monitor) — https://www.litmaps.com/ — MEDIUM confidence -- SciSpace product positioning (chat with PDF / literature review / extract data) — https://www.scispace.com/ — MEDIUM confidence +- **PaperForge internal code inspection:** `paperforge/plugin/main.js` — mode switching architecture (global/paper/collection), per-paper dashboard rendering, contextual buttons, event subscriptions — HIGH confidence +- **PaperForge internal code inspection:** `paperforge/worker/sync.py` — workspace migration creating `ai/` directory (lines 1680-1749) — HIGH confidence +- **PaperForge internal code inspection:** `paperforge/worker/asset_index.py` — canonical index `_build_entry()` creating workspace paths including `ai_path`, `deep_reading_path` — HIGH confidence +- **PaperForge internal code inspection:** `paperforge/skills/literature-qa/scripts/ld_deep.py` — deep-reading scaffold structure, Pass 1/2/3 markers, figure block format — HIGH confidence +- **Obsidian Smart Chat documentation:** Chat thread linking within notes, Dataview dashboards, chat-active/chat-done tracking — https://smartconnections.app/smart-chat/ — HIGH confidence (official plugin docs) +- **Obsidian Copilot (DeepWiki):** Chat persistence and history system — markdown files with YAML frontmatter, session grouping, recent usage tracking — https://deepwiki.com/logancyang/obsidian-copilot/8-chat-persistence-and-history — HIGH confidence (documented architecture) +- **Gemini Scribe chat history guide:** Per-note history file pattern, auto-appending, markdown formatting — https://github.com/allenhutchison/obsidian-gemini — MEDIUM confidence (community plugin docs) +- **Claude Sessions plugin:** Session timeline rendering, summary dashboard with hero cards, Obsidian Bases dashboards — https://github.com/gapmiss/claude-sessions — HIGH confidence (well-documented community plugin) +- **PaulGP llms.txt proposal:** Discussion of AI-readable paper annotations, limitations-first orientation — https://paulgp.com/2026/03/10/llms-txt-for-academic-papers.html — MEDIUM confidence (academic blog post, design philosophy reference) +- **Effortless Academic discussion writing guide:** Q&A-style paper discussion structure — MEDIUM confidence (practitioner guide) + +--- + +*Feature research for: v1.8 AI Discussion Recording & Deep-Reading Dashboard* +*Researched: 2026-05-06* diff --git a/.planning/research/MILESTONE-RESEARCH-LOCK.md b/.planning/research/MILESTONE-RESEARCH-LOCK.md new file mode 100644 index 00000000..772117d8 --- /dev/null +++ b/.planning/research/MILESTONE-RESEARCH-LOCK.md @@ -0,0 +1,97 @@ +# Milestone Research Lock + +**Created:** 2026-05-03 +**Purpose:** Freeze the already-completed milestone-level research conclusions for v1.6 and v1.7 so future workflows do not repeat the same product-direction research. + +--- + +## Scope Of This Lock + +This file records research that is considered settled enough to skip re-research later. + +It does **not** lock implementation details. +It does **not** replace discuss-phase clarification. +It does **not** replace phase planning. + +It only means the product-direction and architecture-direction questions below do not need another milestone-level research pass unless the project direction changes materially. + +--- + +## v1.6 Locked Research + +**Milestone:** AI-Ready Literature Asset Foundation + +### Direction Locked + +- PaperForge should evolve toward long-term literature asset management, not prompt-button sprawl. +- The system should converge around a canonical asset index derived from existing source artifacts. +- Python remains the sole owner of lifecycle, health, maturity, and context-pack logic. +- The plugin remains a thin shell over CLI/index outputs. +- `paper workspace` is the preferred long-term user-facing model, but exact file composition remains a discuss/planning concern, not a re-research concern. +- `ai/` belongs in the paper workspace and stores `/pf-paper`-style atomized, reusable AI notes rather than system caches. +- Complex OCR intermediate artifacts remain system assets unless explicitly promoted into the user workspace. + +### What Does Not Need Re-Research + +- Whether PaperForge should focus on asset management instead of hardcoded extraction tools. +- Whether a canonical index is needed. +- Whether plugin logic should stay thin-shell. +- Whether health/lifecycle/maturity should be centralized. +- Whether the paper workspace direction is valid. + +### What Still Needs Discuss/Planning + +- Exact workspace file names and folder boundaries. +- Whether workspace `fulltext.md` is a mirrored user entry or the only visible copy. +- Exact canonical index schema fields. +- Migration sequencing inside phases 22-26. + +--- + +## v1.7 Locked Research + +**Milestone:** LLMWiki Concept Network + +### Direction Locked + +- `LLMWiki` should be a cross-paper synthesis layer above `Literature/`, not a replacement for it. +- The primary early use case is concept/mechanism network building. +- Wiki outputs must stay source-traceable and reviewable. +- The wiki should compile from canonical assets and curated AI atoms, not raw unstable system byproducts. +- The wiki is a knowledge compilation layer, not a freeform dumping ground and not a domain-specific extraction table factory. + +### What Does Not Need Re-Research + +- Whether v1.7 should introduce an LLM-managed wiki layer. +- Whether the first use case should be concept/mechanism synthesis. +- Whether wiki outputs must be traceable. +- Whether the wiki should be downstream of canonical index + paper workspaces. + +### What Still Needs Discuss/Planning + +- Exact top-level folder name (`LLMWiki/`, `Knowledge/`, etc.). +- Page taxonomy: Topics vs Concepts vs Mechanisms vs Methods. +- Refresh/rebuild policy. +- How atom notes map into wiki nodes and evidence trails. + +--- + +## Workflow Guidance + +For future milestone and phase workflows: + +- **Do not rerun general product-direction research for v1.6 or v1.7 by default.** +- Use these files as upstream context: + - `.planning/research/SUMMARY.md` + - `.planning/research/MILESTONE-RESEARCH-LOCK.md` +- If clarification is needed, handle it in discuss/planning as a design decision, not as a fresh ecosystem research cycle. + +Only reopen research if one of the following changes: + +- PaperForge abandons the Python-first thin-shell architecture. +- The project moves away from local-first literature asset management. +- v1.7 changes from concept/mechanism wiki to a substantially different product direction. + +--- + +*Status: active research lock for v1.6 and v1.7* diff --git a/.planning/research/PITFALLS.md b/.planning/research/PITFALLS.md index cdeac25c..e677f600 100644 --- a/.planning/research/PITFALLS.md +++ b/.planning/research/PITFALLS.md @@ -1,255 +1,625 @@ -# Domain Pitfalls +# Domain Pitfalls — v1.8 AI Discussion & Deep-Reading Dashboard -**Domain:** Brownfield evolution of PaperForge into an AI-ready local-first literature asset manager -**Researched:** 2026-05-03 +**Domain:** Brownfield Obsidian plugin — adding deep-reading dashboard mode and AI discussion recording to existing mode-based routing +**Researched:** 2026-05-06 +**Confidence:** HIGH -## Recommended v1.6 mitigation phases - -1. **Phase 1 — Truth model and migration contract** - - Define source-of-truth ownership for config, user intent, machine facts, and derived state. -2. **Phase 2 — Canonical asset index and rebuild pipeline** - - Build the index as a deterministic projection with repair/rebuild support. -3. **Phase 3 — Health engine and thin-shell dashboard** - - Centralize health/readiness logic in Python; plugin renders only. -4. **Phase 4 — AI context packaging and maturity guidance** - - Add reusable context packs and next-step guidance without hardcoding discipline schemas. -5. **Phase 5 — Brownfield rollout, migration, and verification** - - Ship compatibility checks, doctor/repair upgrades, and safe rollout gates. +--- ## Critical Pitfalls -### Pitfall 1: Creating multiple truths for the same paper -**What goes wrong:** `paperforge.json`, plugin `data.json`, library-record frontmatter, OCR `meta.json`, formal notes, and the new asset index all start carrying overlapping state. Users then see contradictory answers to simple questions like “is this paper OCR-ready?” or “which path is canonical?” +### Pitfall 1: Mode detection ordering regression — `deep-reading.md` hijacks per-paper mode -**Why it happens:** Brownfield systems accrete state by convenience. PaperForge already has proven drift history across queue/status surfaces, and v1.6 adds more derived surfaces unless ownership is locked down first. +**What goes wrong:** +When the user opens `Literature/Orthopedics/ABCDEFG - Biomechanics of ACL Reconstruction/deep-reading.md`, the `_detectAndSwitch()` method encounters a `.md` file, checks for `zotero_key` in frontmatter, and **finds it** — because `deep-reading.md` was generated by the agent with the same `zotero_key` in its frontmatter. The dashboard enters per-paper mode instead of deep-reading mode. The user sees lifecycle stepper instead of Pass 1 summary and AI Q&A history. -**Consequences:** State drift, repair complexity, mistrust in dashboard output, and future migrations that require special-case logic. +**Why it happens:** +The current mode detection logic (main.js lines 718-764) is a linear cascade: +1. No active file → `global` +2. `.base` → `collection` +3. `.md` with `zotero_key` → `paper` +4. `.md` without `zotero_key` → `global` +5. Any other → `global` -**Prevention:** -- Publish a strict ownership matrix before implementation: - - `paperforge.json` = runtime configuration truth - - plugin `data.json` = UI draft/cache only, never workflow truth - - library-record = user intent + stable imported metadata - - OCR/meta outputs = machine facts - - asset index = derived projection only -- Ban duplicate writable fields across layers. -- Add a `source_of_truth` note to ADR/docs for every new field. +There is no check for the *filename* being `deep-reading.md` before checking for `zotero_key`. Since `deep-reading.md` carries `zotero_key` in its frontmatter (for traceability), the paper mode branch wins. -**Detection:** -- Same concept appears writable in both plugin and CLI outputs -- “Fix” commands need to choose which file wins -- Status differs depending on command/UI surface +**How to avoid:** +Insert the deep-reading check as the **first branch** inside the `.md` handler, before the `zotero_key` check: -**Mitigation phase:** Phase 1 +```js +if (ext === 'md') { + // Check for deep-reading first (before zotero_key check) + if (activeFile.basename === 'deep-reading' || activeFile.path.includes('/deep-reading.md')) { + const cache = this.app.metadataCache.getFileCache(activeFile); + const key = cache?.frontmatter?.zotero_key; + if (key) { + this._currentPaperKey = key; + this._currentPaperEntry = this._findEntry(key); + this._currentDomain = null; + this._switchMode('deep-reading'); + return; + } + } + // ... existing zotero_key check ... + const cache = this.app.metadataCache.getFileCache(activeFile); + const key = cache?.frontmatter?.zotero_key; + if (key) { /* paper mode */ } +} +``` -### Pitfall 2: Treating the canonical asset index like a hand-maintained database -**What goes wrong:** The new index becomes a semi-manual store that mixes imported facts, repair flags, health summaries, and UI-only annotations. Once users or multiple commands edit it directly, rebuilds stop being trustworthy. +The ordering discipline is: **most specific filename pattern wins.** `deep-reading.md` is more specific than "any `.md` with `zotero_key`." -**Why it happens:** “Canonical index” sounds like “put everything there.” In local-first systems, durable tables often become junk drawers unless they are clearly defined as projections. +**Warning signs:** +- User opens `deep-reading.md` and sees lifecycle stepper instead of Pass 1 summary +- Header shows "Paper" badge instead of "Deep Reading" badge +- `_renderPaperMode()` is called when `_renderDeepReadingMode()` should be -**Consequences:** Non-idempotent rebuilds, hard-to-debug corruption, impossible provenance, and brittle plugin/CLI dependencies. +**Phase to address:** Phase 1 (Deep-reading dashboard mode) -**Prevention:** -- Make the index a deterministic projection from owned sources, not a primary authoring surface. -- Store per-record provenance: input files, mtimes/hashes, generated_at, schema_version. -- Support `rebuild-index` from source artifacts with no data loss. -- Keep manual overrides in a separate override layer if truly needed. +--- -**Detection:** -- Rebuilding the index changes meaning, not just freshness -- Developers hesitate to delete/regenerate the index -- User edits to the index are considered normal workflow +### Pitfall 2: `fs.readFileSync` for `discussion.json` bypasses Obsidian vault API — missing cache invalidation and encoding issues -**Mitigation phase:** Phase 2 +**What goes wrong:** +The plugin reads `discussion.json` via Node.js `fs.readFileSync()` (same pattern as `_fetchStats` and `_loadIndex` for `formal-library.json`). This works for `formal-library.json` because it lives outside the Obsidian vault timeline in `<system_dir>/PaperForge/indexes/`. But `discussion.json` lives in the paper workspace `ai/` directory which IS inside the vault. Using `fs` directly means: +1. Obsidian's metadata cache doesn't know about the read +2. Vault `modify` events for `discussion.json` won't trigger dashboard refresh +3. On Windows with CJK locales, file content written by Python may be GBK-encoded on disk but read by Node.js as `utf-8`, causing mojibake for Chinese Q&A content -### Pitfall 3: Mixing user intent, machine facts, and derived readiness in one state machine -**What goes wrong:** Fields like `analyze`, `do_ocr`, `ocr_status`, deep-reading completion, figure readiness, and AI readiness get collapsed into one monolithic lifecycle enum. A user toggle then accidentally overwrites machine facts, or a failed OCR run blocks unrelated workflows forever. +**Why it happens:** +The plugin already has a pattern of `fs.readFileSync()` for indices (line 322, 428), making it natural to extend the pattern. But `discussion.json` is fundamentally different — it's inside the vault (paper workspace → visible in Obsidian file explorer) while `formal-library.json` is in a system directory (invisible, machine-controlled). -**Why it happens:** A single “state” field feels tidy, but PaperForge already has different actors: user, worker, agent, and derived health logic. +Additionally, `child_process.spawn('python')` on Windows can produce GBK-encoded stdout/stderr when the system code page is CP936. If discussion content is generated by Python (`/pf-paper` via `ld_deep.py`), the file written to disk might use the system encoding rather than explicit UTF-8. -**Consequences:** Sticky bad states, confusing retries, complicated repair code, and unreadable business logic. +**How to avoid:** -**Prevention:** -- Separate three planes explicitly: - - **Intent:** what the user wants next - - **Asset facts:** what files/processes actually exist - - **Derived readiness/health:** computed conclusions -- Model readiness as computed predicates, not stored toggles. -- Only persist state transitions owned by a real actor. +Layer 1 — Read strategy: +- For `discussion.json` (vault-internal), use Obsidian `app.vault.adapter.read()` instead of `fs.readFileSync()` to ensure vault cache consistency +- For `discussion.md` (human-readable), use the same pattern +- Keep `fs.readFileSync()` only for `formal-library.json` and other system-directory files outside the vault -**Detection:** -- One command both changes user intent and marks derived readiness -- Retry flows require manually editing multiple unrelated fields -- “Why is this blocked?” cannot be answered from one explanation chain +Layer 2 — Encoding hardening: +- In the Python `discussion.json` writer, always use `encoding='utf-8'` explicitly: + ```python + with open(discussion_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + ``` +- In the Python child process spawn, inject UTF-8 environment variables: + ```python + env = os.environ.copy() + env['PYTHONIOENCODING'] = 'utf-8' + env['PYTHONUTF8'] = '1' + ``` +- In the plugin's Node.js reads, always pass explicit `'utf-8'` encoding: + ```js + const raw = this.app.vault.adapter.read(discussionPath); + // or if using fs: + const raw = fs.readFileSync(discussionPath, 'utf-8'); + ``` -**Mitigation phase:** Phase 1 +Layer 3 — Event subscription: +- Add `discussion.json` paths to the vault `modify` event filter alongside `formal-library.json` (line 1259): + ```js + const modifyHandler = this.app.vault.on('modify', (file) => { + if (file?.path?.endsWith('formal-library.json') || file?.path?.endsWith('discussion.json')) { + this._invalidateIndex(); + this._refreshCurrentMode(); + } + }); + ``` -### Pitfall 4: Re-implementing health and lifecycle logic inside the Obsidian plugin -**What goes wrong:** The plugin starts recomputing health, maturity, and readiness in JavaScript because it needs fast dashboard cards. Soon the Python CLI and plugin disagree about counts, warnings, and next steps. +**Warning signs:** +- Chinese Q&A text in dashboard shows garbled characters (mojibake) +- Dashboard doesn't update after Python writes new discussion entries +- `JSON.parse()` throws `Unexpected token` errors on apparently valid Chinese content +- Console shows file reads from `fs` with byte-level corruption visible in hex dump -**Why it happens:** UI teams want instant rendering, and dashboard work invites “just compute it here.” But PaperForge already established a thin-shell direction precisely to avoid this split. +**Phase to address:** Phase 2 (AI discussion recorder) + Phase 3 (Integration testing) -**Consequences:** Plugin/CLI duplication, inconsistent support burden, doubled test surface, and brownfield regressions whenever state rules change. +--- -**Prevention:** -- Expose health/index data from Python as stable JSON contracts. -- Plugin consumes rendered summaries or structured check results; it does not infer workflow truth. -- Put contract fixtures in tests used by both CLI and plugin. -- Refuse UI-only derived fields unless they are clearly cosmetic. +### Pitfall 3: `discussion.json` schema evolution without versioning — silent data loss on format change -**Detection:** -- Same rule exists in JS and Python -- Dashboard numbers differ from `paperforge status` -- Plugin release requires business-rule changes without CLI changes +**What goes wrong:** +The first implementation of `discussion.json` uses a simple array-of-objects format: +```json +[ + {"timestamp": "2026-05-06T10:00:00Z", "role": "user", "content": "What is the main finding?"}, + {"timestamp": "2026-05-06T10:00:05Z", "role": "assistant", "content": "The main finding is..."} +] +``` -**Mitigation phase:** Phase 3 +Three months later, the format needs to add `model`, `tokens_used`, `source_sections`, or a `conversation_id`. Existing `discussion.json` files break in the dashboard. There is no `schema_version` field to trigger a migration path. -### Pitfall 5: Building maturity scores that look smart but are operationally empty -**What goes wrong:** “Library maturity” becomes a vanity number with no evidence trail. Users see 62/100 but cannot tell what to fix next, or why the score changed after a sync. +**Why it happens:** +The `formal-library.json` schema evolution (adding `schema_version: "2"`) was learned painfully in v1.6/v1.7. But `discussion.json` is a new artifact created in v1.8 — the team may repeat the same mistake thinking "it's just a simple flat file." -**Why it happens:** Product scoring is tempting, especially for AI readiness, but scoring before explainability creates opaque UX. +**How to avoid:** +- Ship `discussion.json` with `schema_version: "1"` in the envelope from day one: + ```json + { + "schema_version": "1", + "generated_at": "2026-05-06T10:00:00Z", + "paper_key": "ABCDEFG", + "conversations": [ + {"id": "conv_001", "timestamp": "...", "entries": [...]} + ] + } + ``` +- Add a tolerant reader in the plugin that checks `schema_version` and falls back gracefully: + ```js + function parseDiscussion(raw) { + const data = JSON.parse(raw); + if (!data.schema_version) { + // Legacy format: raw array + return { schema_version: "0", conversations: [{ entries: data }] }; + } + if (data.schema_version === "1") return data; + console.warn('PaperForge: Unknown discussion schema version', data.schema_version); + return null; + } + ``` +- Never use top-level array (`[...]`) as the root JSON type. Always use an object envelope (`{"schema_version": "...", ...}`) to allow backward-compatible field additions. -**Consequences:** User distrust, noisy support requests, and pressure to game the score instead of improving library quality. +**Warning signs:** +- "Add one more field" request for `discussion.json` triggers format debate +- Dashboard code has `if (Array.isArray(discussion))` branches +- JSON.parse throws on older files after format change +- Users report "AI discussion history disappeared" after plugin update -**Prevention:** -- Every score must decompose into named checks with weights, evidence, and recommended next action. -- Prefer level-based maturity bands at first (`Foundational`, `Usable`, `AI-ready`) over fake precision. -- Show “what improved / what regressed” deltas after worker runs. +**Phase to address:** Phase 2 (AI discussion recorder) — design the schema first, implement write + read together -**Detection:** -- Score exists without per-check explanations -- Two libraries with different failure modes get the same score -- Users ask “what does this number mean?” +--- -**Mitigation phase:** Phase 4 +### Pitfall 4: `btoa()` / `atob()` on Chinese filenames in discussion file paths -### Pitfall 6: Over-productizing extraction schemas into the core asset model -**What goes wrong:** v1.6 turns “AI-ready” into built-in PICO tables, mechanism schemas, parameter schemas, or discipline-specific extraction templates embedded in the canonical index. +**What goes wrong:** +The plugin needs to resolve the path to `discussion.json` from the paper's `zotero_key` and `title`. If the title contains Chinese characters (e.g., "膝关节置换术后康复效果分析"), and any path resolution code uses `btoa()` or `encodeURIComponent()` followed by regex operations, the code will throw `InvalidCharacterError: Failed to execute 'btoa' on 'Window': The string to be encoded contains characters outside of the Latin1 range.` -**Why it happens:** Once context packaging exists, the fastest demo is a hardcoded schema. But the project explicitly wants a framework, not a stack of medical extraction products. +**Why it happens:** +`window.btoa()` only supports Latin1 characters (U+0000 to U+00FF). Chinese characters fall outside this range. This is a well-known pattern of failure in Obsidian plugins (documented in opencode-obsidian Issue #28, confirmed by multiple Obsidian plugin developers). -**Consequences:** Schema lock-in, brittle migrations, domain overfitting, poor reuse outside one workflow, and a permanently bloated index contract. +PaperForge's current codebase does NOT use `btoa()` directly, but future path construction for `ai/` directory resolution might introduce it, especially if a content-hash or Base64 slug is used for file naming. -**Prevention:** -- Keep the core asset model generic: provenance, text assets, figures, notes, readiness, health. -- Implement context packs as pluggable packagers/templates over the core assets. -- Allow optional schema manifests outside the canonical index. -- Treat missing structured extraction as a pack-level capability gap, not a library corruption. +**How to avoid:** +- **Ban `btoa()` and `atob()`** from the plugin codebase. If Base64 encoding is needed, use Node.js `Buffer`: + ```js + // SAFE: Node.js Buffer handles all Unicode + const encoded = Buffer.from(chineseString, 'utf-8').toString('base64'); + const decoded = Buffer.from(encoded, 'base64').toString('utf-8'); + ``` +- For path construction, use filesystem-safe slugs derived from `zotero_key` only (which is always ASCII), never from `title`. +- Encode this rule in a linter or pre-commit check: `grep -r 'btoa(' paperforge/plugin/` must return empty. -**Detection:** -- Core index fields start naming domain-specific extraction slots -- New workflows require core schema migration instead of a new packager -- “AI-ready” is defined as “matches one extraction template” +**Warning signs:** +- Plugin crashes with `InvalidCharacterError` in console when paper title contains Chinese +- Only reproduces on papers with non-ASCII titles +- Error traceback points to `btoa` or `atob` in a path-building function -**Mitigation phase:** Phase 4 +**Phase to address:** Phase 2 (AI discussion recorder) — path resolution design -### Pitfall 7: Shipping v1.6 without a brownfield migration and rollback story -**What goes wrong:** Existing vaults upgrade into new index/state logic with partial migrations, stale Bases, old commands, and no clean recovery path. +--- -**Why it happens:** Teams focus on the new model, not on the installed base. PaperForge already has real-world path quirks, metadata drift, and existing plugin persistence. +### Pitfall 5: `active-leaf-change` double-firing creates mode oscillation between per-paper and deep-reading -**Consequences:** Broken dashboards, false health alarms, user-edited records rendered invalid, and high-maintenance emergency fixes. +**What goes wrong:** +When the user clicks from `ABCDEFG - Title.md` (per-paper mode) to `deep-reading.md` (deep-reading mode), the Obsidian workspace fires `active-leaf-change` **twice** (a known Obsidian API behavior — see forum thread #31841). The first firing has the old leaf context, the second has the new. If the 300ms debounce (line 1251) fires between the two events, the dashboard briefly switches to `global` mode (no active file detected) before switching to `deep-reading` mode. This causes a visible flash of the global dashboard. -**Prevention:** -- Version every generated artifact and contract. -- Add `doctor` checks for config drift, index schema version, stale Base templates, and orphaned OCR/meta assets. -- Add one-step repair/rebuild commands before turning on new dashboard surfaces. -- Keep rollout reversible: regenerate index, restore previous Bases, ignore unsupported plugin fields gracefully. +Additionally, if both `deep-reading.md` and the formal note are open in split panes, the mode will oscillate between `paper` and `deep-reading` on every focus change. -**Detection:** -- Upgrade requires manual file surgery -- Users must delete unknown files to recover -- Docs say “reinstall if weird things happen” +**Why it happens:** +- `active-leaf-change` event is known to fire twice during tab switches (old leaf blur + new leaf focus) +- The 300ms debounce can catch either one depending on timing +- The mode switching logic (`_switchMode`) has an early return `if (this._currentMode === mode)`, but doesn't distinguish between "same mode, different file" and "new mode for same paper" -**Mitigation phase:** Phase 5 +**How to avoid:** +1. **Guard with identity check, not string comparison:** + ```js + _detectAndSwitch() { + const activeFile = this.app.workspace.getActiveFile(); + // Track the file that triggered current mode + const newModeAndFile = this._resolveModeForFile(activeFile); + if (this._currentMode === newModeAndFile.mode && + this._activeFilePath === (activeFile?.path || null)) { + return; // Same mode AND same file — no-op + } + // Different file or different mode — switch + this._activeFilePath = activeFile?.path || null; + this._switchMode(newModeAndFile.mode, newModeAndFile.context); + } + ``` +2. **Extract `_resolveModeForFile()` as pure function:** + - Returns `{ mode: 'paper'|'deep-reading'|'collection'|'global', context: {...} }` + - No side effects, fully testable + - Called by both `_detectAndSwitch()` and the debounced handler +3. **Increase debounce to 500ms** during mode transitions (from current 300ms) to absorb the double-fire + +**Warning signs:** +- Dashboard flashes global mode briefly when switching to deep-reading +- `_switchMode` is called 2-3 times for a single user click +- Console logs show rapid mode sequence: `paper → global → deep-reading` or `deep-reading → paper → deep-reading` +- Split-pane usage causes dashboard to flicker between modes + +**Phase to address:** Phase 1 (Deep-reading dashboard mode) + +--- + +### Pitfall 6: Empty `discussion.json` / missing `ai/` directory breaks dashboard rendering + +**What goes wrong:** +When the dashboard enters deep-reading mode, it attempts to read `discussion.json` from the paper's `ai/` directory. For papers that haven't had any AI discussions yet: +1. `ai/discussion.json` doesn't exist → `JSON.parse()` throws on undefined/null +2. `ai/` directory doesn't exist → `fs.readFileSync()` throws `ENOENT` +3. `discussion.json` exists but is empty (0 bytes) → `JSON.parse('')` throws `Unexpected end of JSON input` +4. `discussion.json` has `conversations: []` → dashboard should show "No AI discussions yet" empty state + +The current codebase has good empty state handling via `_renderEmptyState()` (line 405-409) and `_renderSkeleton()` (line 399-401), but only for the global/paper/collection modes. The deep-reading mode renderer must handle all four empty states explicitly. + +**Why it happens:** +Dashboard mode renderers often assume data exists. The first implementation skips the "file doesn't exist" check because the developer tested with a paper that already had discussions. + +**How to avoid:** +- Write a `_loadDiscussionData(key)` helper that returns a discriminated union: + ```js + _loadDiscussionData(key) { + const discussionPath = this._resolveDiscussionPath(key); + if (!discussionPath) return { status: 'no_ai_dir' }; + try { + const raw = fs.readFileSync(discussionPath, 'utf-8'); + if (!raw.trim()) return { status: 'empty' }; + const data = JSON.parse(raw); + if (!data.conversations || data.conversations.length === 0) { + return { status: 'no_discussions' }; + } + return { status: 'ok', data }; + } catch (e) { + if (e.code === 'ENOENT') return { status: 'not_found' }; + return { status: 'parse_error', error: e.message }; + } + } + ``` +- Render distinct UI for each status: + - `no_ai_dir` / `not_found`: "This paper hasn't been discussed with AI yet. Use /pf-paper or /pf-deep to start." + - `empty`: Same as above (file exists but empty = never started) + - `no_discussions`: Same as above (conversations array is empty) + - `parse_error`: "Discussion data is corrupted. Re-run /pf-paper to regenerate." + - `ok`: Render Q&A history with Pass 1 summary + +**Warning signs:** +- Dashboard shows raw error text for new papers without AI discussions +- `Uncaught TypeError: Cannot read property 'conversations' of null` in console +- Users think AI discussion feature is broken when actually it's just never been used + +**Phase to address:** Phase 1 (Deep-reading dashboard mode) + Phase 2 (AI discussion recorder) — both must handle empty states + +--- + +### Pitfall 7: "Jump to Deep Reading" button assumes `deep-reading.md` exists at a fixed path + +**What goes wrong:** +The per-paper dashboard card renders a "Jump to Deep Reading" button that navigates to `Literature/<domain>/<key> - <Title>/deep-reading.md`. But: +1. Deep reading may not have been performed yet → file doesn't exist → `openLinkText` silently fails or opens an empty new note +2. The paper title may have changed → path is stale +3. The `domain` may have changed → path is stale +4. The `key` is correct but the file is in a different location (vault restructured) + +**Why it happens:** +Path construction from canonical index fields (`entry.domain`, `entry.title`, `key`) assumes those fields are immutable and the file structure is static. But Zotero metadata updates, title normalization, or vault reorganization can break the assumption. + +**How to avoid:** +- Store the `deep_reading_note_path` in the canonical index (`formal-library.json`) as a machine-maintained field, similar to how `fulltext_path` is already stored +- Before navigating, verify file existence: + ```js + _openDeepReading(key) { + const entry = this._findEntry(key); + if (!entry?.deep_reading_note_path) { + new Notice('Deep reading not yet generated for this paper. Run /pf-deep first.', 6000); + return; + } + const file = this.app.vault.getAbstractFileByPath(entry.deep_reading_note_path); + if (!file) { + new Notice('Deep reading file not found. It may have been moved or deleted.', 6000); + return; + } + this.app.workspace.openLinkText(file.path, ''); + } + ``` +- Add `deep_reading_note_path` to the canonical index schema in the `paths` section +- Update the index builder to populate this field from the file system (check if `deep-reading.md` exists in the paper workspace) + +**Warning signs:** +- Button navigation opens wrong file or nothing happens +- Users report "Jump to Deep Reading does nothing" but only on specific papers +- Console shows `getAbstractFileByPath` returning null +- Button is shown even when deep reading hasn't been performed + +**Phase to address:** Phase 1 (Deep-reading dashboard mode) + +--- + +### Pitfall 8: Version number display race condition — plugin loads before Python reports version + +**What goes wrong:** +The version badge (`v—` placeholder, line 281) is populated from `_fetchStats()` which reads `formal-library.json` or falls back to `python -m paperforge status --json`. The badge starts as `v—` (em dash placeholder) and only updates after the first successful stats fetch. If `formal-library.json` is missing (fresh install, no sync yet), AND the Python fallback fails (Python not installed, `paperforge` not installed), the version stays as `v—` permanently. + +**Why it happens:** +The version is derived from runtime data, not from static plugin metadata. The plugin has its own version in `manifest.json` but doesn't use it because the plugin version != the Python package version. The Python package version is authoritative (only it knows the CLI features available), but reaching it requires a subprocess. + +**How to avoid:** +- **Primary source:** Read version from `paperforge/__init__.py` or `paperforge/plugin/manifest.json` at plugin load time via a fast synchronous fallback file +- **Show version badge immediately** with plugin manifest version as a floor: + ```js + // In onOpen(): + this._versionBadge = headerLeft.createEl('span', { ... }); + // Try to get Python version immediately + this._loadVersion(); + + _loadVersion() { + // Fast path: read from a cached version file written by setup + const vp = this.app.vault.adapter.basePath; + const versionPath = path.join(vp, systemDir, 'PaperForge', 'indexes', '.version'); + try { + const v = fs.readFileSync(versionPath, 'utf-8').trim(); + if (v) { this._versionBadge.setText('v' + v); return; } + } catch {} + // Fallback: spawn python -m paperforge --version (fast, single-line output) + exec('python -m paperforge --version', { cwd: vp, timeout: 5000 }, (err, stdout) => { + if (!err && stdout.trim()) { + this._versionBadge.setText('v' + stdout.trim()); + } + }); + } + ``` +- The `_fetchStats()` should update the version only if it's newer/more specific than what's already shown (never step backward) + +**Warning signs:** +- Version badge shows `v—` even after successful sync +- Version badge flickers between `v—` and `v1.8.0` during mode switches +- `_renderStats()` resets version to `—` on error paths (line 359: `version: this._cachedStats?.version || '\u2014'`) + +**Phase to address:** Phase 4 (Bug fixes) + +--- + +### Pitfall 9: Removing "ai" UI row without checking all render paths + +**What goes wrong:** +The bug fix says "remove meaningless 'ai' row." But the rendering code has multiple paths: +1. `_renderGlobalMode()` — renders OCR section only +2. `_renderPaperMode()` — renders paper view +3. `_renderCollectionMode()` — renders collection view +4. `_renderModeHeader()` — renders header badge +5. Settings tab display — shows config summary + +If "ai" refers to a row in the settings config summary (line 1397), removing it from one path but not others leaves a partial fix. If "ai" is an action card in `ACTIONS` (line 157-208), removing the `paperforge-copy-context` entry would break the contextual action buttons in `_renderPaperMode()`. + +**Why it happens:** +Bug fix descriptions are under-specified. "Remove the 'ai' row" could mean: +- Remove "Copy Context" action (but it's used in per-paper mode) +- Remove a config summary row showing `ai/` directory path +- Remove some other "ai" label + +Without tracing all render paths, the fix risks breaking dependent features. + +**How to avoid:** +- Before making changes, grep for "ai" across all plugin JS, CSS, and i18n strings: + ```bash + grep -n -i '"ai"' paperforge/plugin/main.js + grep -n -i 'ai' paperforge/plugin/styles.css + ``` +- Identify which specific UI element is the "meaningless ai row" +- Create a mini-checklist of all rendering paths that contain or reference "ai" +- After removal, verify no other features broke (Copy Context still works, per-paper actions still render) +- Add a CSS class scope guard so removal is surgical, not global + +**Warning signs:** +- After the fix, Copy Context buttons disappear from per-paper mode +- Config summary in settings tab loses rows it shouldn't +- User reports "Some buttons are gone" after update + +**Phase to address:** Phase 4 (Bug fixes) — must be the LAST fix applied after all features are stable + +--- ## Moderate Pitfalls -### Pitfall 8: Health checks that are too shallow to be trusted -**What goes wrong:** “PDF Health” only checks path existence, “OCR Health” only checks folder presence, and “Base Health” only checks file existence. Users get green badges on unusable assets. +### Pitfall 10: `discussion.md` Q&A format inconsistency between human-readable and machine-parseable + +**What goes wrong:** +`discussion.md` is designed as human-readable Q&A format. `discussion.json` is the machine-consumable version. If they disagree (extra entries in `.md`, missing entries in `.json`, different timestamps), the dashboard shows one thing but the markdown file shows another. The user becomes confused about which is authoritative. **Prevention:** -- Validate at the failure mode level: file exists, readable, non-empty, expected sidecars present, schema/version matches, referenced note exists, figure-map parseable. -- Distinguish `missing`, `broken`, `stale`, and `incomplete`. -- Return evidence paths, not just booleans. +- Define `discussion.json` as the **canonical source** for dashboard rendering +- Define `discussion.md` as a **derived view** (always generated from `.json`) +- Never write to `.md` without writing to `.json` first +- Add a validation check in `doctor` that compares `.md` and `.json` consistency +- **Do not support** "hand-edited `discussion.md`" as a workflow — the `.json` always wins -**Mitigation phase:** Phase 3 +**Mitigation phase:** Phase 2 (AI discussion recorder) -### Pitfall 9: Making AI context packs too heavy, too eager, or too opaque -**What goes wrong:** `ask-this-paper` or `copy-context-pack` assembles huge prompt blobs every time, includes duplicated OCR/note content, and hides where statements came from. +### Pitfall 11: Tabs/spaces in `discussion.md` breaking Obsidian callout rendering + +**What goes wrong:** +Python generates `discussion.md` with Q&A entries formatted as Obsidian callouts: +```markdown +> [!question]- What is the main finding of this paper? +> The paper found that ACL reconstruction using hamstring autograft... +``` + +If Python uses tabs for indentation, or the markdown has inconsistent newline handling (Windows `\r\n` vs Unix `\n`), Obsidian may not render the callout correctly. Chinese text with certain punctuation at line boundaries can also break callout syntax. **Prevention:** -- Build packs from explicit sections with token budgets. -- Prefer manifest-first packaging: summary + citations + selected assets + provenance links. -- Cache pack manifests; assemble full text lazily. -- Include source paths for every included block. +- Use `\n` consistently (Python writes with `newline='\n'` even on Windows) +- Avoid tabs in generated markdown content +- Test callout rendering with Chinese text containing full-width punctuation (`。`, `，`, `：`) +- Provide a regex-based callout validator that catches common formatting issues -**Mitigation phase:** Phase 4 +**Mitigation phase:** Phase 2 (AI discussion recorder) -### Pitfall 10: Dashboard-first implementation order -**What goes wrong:** The plugin dashboard is built before the index and health contracts stabilize, so UI demands freeze bad backend assumptions. +### Pitfall 12: Debounce timer leak when deep-reading mode content refreshes frequently + +**What goes wrong:** +If the Python agent is actively writing to `discussion.json` (streaming Q&A entries during an active `/pf-deep` session), the vault `modify` event fires for each write. The current 300ms debounce and mode refresh cycle will repeatedly re-render the deep-reading dashboard, causing: +1. UI jank (constant re-rendering) +2. Flicker of the "switching" CSS class +3. Lost scroll position in the Q&A history view +4. Increased CPU usage from repeated `JSON.parse()` of growing `discussion.json` **Prevention:** -- Finish JSON contracts and fixture snapshots before dashboard polish. -- Use a temporary debug/JSON view during backend stabilization. -- Only add cards/recommendations after health semantics are verified. +- Add a **cooldown period** after a refresh: don't refresh again within 2 seconds of the last refresh +- Use a **dirty flag** pattern: mark data as stale but defer re-render until user-visible idle +- For `discussion.json`, use a `mtime` comparison: only re-render if the file modification time actually changed (not just any `modify` event) +- Consider `requestIdleCallback` or `setTimeout(fn, 1000)` for non-critical refreshes -**Mitigation phase:** Phase 3 +**Mitigation phase:** Phase 3 (Integration testing & performance) -### Pitfall 11: Ignoring stale generated surfaces beyond the new index -**What goes wrong:** The index updates, but generated Bases, note templates, cached plugin views, and old dashboard expectations still reflect pre-v1.6 fields. +### Pitfall 13: CSS class namespace collision with new deep-reading dashboard classes + +**What goes wrong:** +The current CSS file (`styles.css`) is 1325 lines with sections 1-17. Adding deep-reading dashboard CSS (section 18+) must not accidentally override existing classes like `.paperforge-paper-view`, `.paperforge-header`, or `.paperforge-metric-card`. If a new selector is too broad (e.g., `.paperforge-content-area div`), it will affect the global, paper, and collection mode renders. **Prevention:** -- Track generator version on every generated artifact. -- Add stale-template/Base detection to doctor. -- Make regeneration explicit and safe. +- Use **strictly scoped class names**: `paperforge-deepreading-*` prefix for all deep-reading dashboard CSS +- Avoid inheritance-based selectors; prefer explicit class chaining +- Add a **mode-specific wrapper class** on the content area: `this._contentEl.addClass('paperforge-mode-deepreading')` +- Scope all deep-reading styles under `.paperforge-mode-deepreading` +- Run a CSS diff against previous version: ensure no existing class definitions were modified -**Mitigation phase:** Phase 5 +**Mitigation phase:** Phase 1 (Deep-reading dashboard mode) + +--- ## Minor Pitfalls -### Pitfall 12: Vocabulary drift across health, readiness, and maturity terms -**What goes wrong:** “ready,” “healthy,” “complete,” “indexed,” and “AI-ready” are used inconsistently across CLI, plugin, docs, and Bases. +### Pitfall 14: Hardcoded `ai/` directory path in plugin JS + +**What goes wrong:** +The plugin hardcodes `ai/` as the discussion directory, but the actual directory is configurable via `paperforge.json` path settings. If a user changes the workspace layout, the plugin can't find discussion files. **Prevention:** -- Publish a term glossary with exact meanings. -- Reuse the same labels in CLI JSON, plugin cards, docs, and Base columns. +- Use the workspace path from `paperforge.json` (already loaded by `readPaperforgeJson()`) +- The `ai/` directory is always `{paper_workspace}/ai/`, and the paper workspace path comes from canonical index `paths.workspace` +- Never concatenate string segments manually; use a path resolver function -**Mitigation phase:** Phase 1 +**Mitigation phase:** Phase 2 (AI discussion recorder) -### Pitfall 13: Hiding actionable repair paths behind aggregate status -**What goes wrong:** Users see “12 broken assets” but not the exact records or command to fix them. +### Pitfall 15: `discussion.md` numbering conflicts with existing markdown headings in formal notes + +**What goes wrong:** +If `discussion.md` uses `##` headings that match formal note headings (e.g., `## Methods`, `## Results`), and the user embeds `discussion.md` into the formal note via Obsidian `![[embedded]]` syntax, heading collisions cause broken navigation in Obsidian's outline view. **Prevention:** -- Every aggregate status should drill down to paper-level evidence and a fix path. -- CLI and plugin should share the same remediation text. +- Use a unique heading prefix for discussion entries: `## 💬 AI Discussion` or `## 🤖 Agent Q&A` +- Document the expected embedding pattern in AGENTS.md +- Keep discussion headings distinct from paper content headings -**Mitigation phase:** Phase 3 +**Mitigation phase:** Phase 2 (AI discussion recorder) -## Phase-Specific Warnings +--- -| Phase Topic | Likely Pitfall | Mitigation | -|-------------|---------------|------------| -| Truth model | Field ownership ambiguity between config, plugin cache, library-record, OCR meta, and index | Publish ownership matrix and forbid duplicate writable fields | -| Canonical index | Index becomes a second database instead of a rebuildable projection | Deterministic projection + provenance + rebuild command | -| Lifecycle model | Intent/facts/readiness collapsed into one enum | Separate planes and compute readiness | -| Health engine | Green badges based on existence-only checks | Failure-mode-level checks with evidence | -| Plugin dashboard | JS duplicates Python business rules | Python emits JSON contracts; plugin renders only | -| Maturity scoring | Opaque score with no next action | Explainable checks, weighted bands, remediation text | -| AI context packaging | Core model polluted by discipline-specific schemas | Keep packagers optional and outside core index | -| Brownfield rollout | Old vaults, Bases, and caches break on upgrade | Versioned artifacts, doctor/repair, reversible rebuild | +## Technical Debt Patterns + +| Shortcut | Immediate Benefit | Long-term Cost | When Acceptable | +|----------|-------------------|----------------|-----------------| +| Use `fs.readFileSync` for `discussion.json` instead of `vault.adapter.read` | Faster, no async overhead | Vault cache inconsistency, missed modify events | Never — discussion.json is vault-internal, use vault API | +| Skip `schema_version` on `discussion.json` (bare array format) | Saves 3 lines of code | Every format change = breaking change | Never — add envelope on first write | +| Hardcode `300ms` debounce everywhere | Simple, works on fast machines | Double-fire of `active-leaf-change` causes mode oscillation on slower systems | Only if tested on representative hardware (HDD, low RAM) | +| Duplicate mode detection logic in `_detectAndSwitch` and debounced handler | Saves extracting a pure function | Bug fix in one place breaks the other | Never — extract `_resolveModeForFile()` from day one | +| Skip empty state handling for new papers | Dashboard renders fine for "test" paper with discussions | Production users see raw errors on first use | Never — handle all four empty states in initial implementation | + +--- + +## Integration Gotchas + +| Integration | Common Mistake | Correct Approach | +|-------------|----------------|------------------| +| Python → JSON file → Node.js read | Python writes GBK, Node.js reads as UTF-8 → mojibake | Python: `open(path, 'w', encoding='utf-8')`. Node.js: `readFileSync(path, 'utf-8')` with explicit encoding | +| Vault `modify` event → dashboard refresh | Subscribing to ALL modifications → dashboard re-renders on every keystroke | Filter to `discussion.json` and `formal-library.json` only; add mtime check to avoid redundant re-renders | +| Canary index `paths` → file navigation | Computing paths from `domain + title + key` → breaks on rename | Store resolved paths in canonical index; verify existence before navigating | +| Obsidian `openLinkText` for deep-reading file | Calling without existence check → silent failure or empty note creation | Check `getAbstractFileByPath` first; show clear Notice on missing file | +| `active-leaf-change` → mode detection | Handling raw event without debounce or dedup → mode oscillation | Debounce 500ms + identity check on file path + mode string before committing to switch | + +--- + +## Performance Traps + +| Trap | Symptoms | Prevention | When It Breaks | +|------|----------|------------|----------------| +| Re-parsing `discussion.json` on every mode refresh | Dashboard lags during active `/pf-deep` session (Python writes Q&A entries rapidly) | Cache parsed `discussion.json` with mtime; only re-parse if mtime changed | 10+ Q&A entries, or slow disk | +| Reading `discussion.md` for dashboard (human-readable, large) | Dashboard load time increases with discussion length | Dashboard reads `.json` only (structured, compact); `.md` is for human reading in Obsidian editor | 50+ Q&A exchanges | +| `_refreshCurrentMode()` on every `modify` event for `discussion.json` | Constant UI rebuild during streaming writes | Add 2-second cooldown after refresh; use dirty flag pattern | Active AI agent writing to discussion | +| Iterating ALL items in canonical index on each deep-reading mode switch | Finding paper entry scans full index array | Use the already-cached `_currentPaperEntry` from mode detection (no re-scan needed) | 1000+ papers in library | + +--- + +## "Looks Done But Isn't" Checklist + +- [ ] **Deep-reading mode:** Opens `deep-reading.md` — verify mode badge says "Deep Reading" not "Paper" +- [ ] **Deep-reading mode:** Opens `deep-reading.md` when BOTH formal note and deep-reading note are open in split panes — verify correct mode per active pane +- [ ] **Deep-reading mode:** Opens `deep-reading.md` for a paper that has NO AI discussions — verify empty state renders, no error +- [ ] **Deep-reading mode:** Opens `deep-reading.md` for a paper with Chinese title — verify Pass 1 summary renders correctly without mojibake +- [ ] **Discussion recording:** Python writes `discussion.json` with Chinese Q&A — verify Node.js reads and displays correctly +- [ ] **Discussion recording:** Python writes `discussion.json` while dashboard is open — verify dashboard refreshes without flicker +- [ ] **Jump button:** Click "Jump to Deep Reading" on a paper that hasn't had deep reading — verify error Notice, no silent failure +- [ ] **Jump button:** Click "Jump to Deep Reading" on a paper that HAS deep reading — verify file opens correctly +- [ ] **Version badge:** Plugin loads with no `paperforge` installed — verify `v—` or plugin manifest version shown, not crash +- [ ] **Version badge:** After sync completes, version is populated from Python — verify it doesn't revert to `v—` on mode switch +- [ ] **"ai" row removal:** All render paths (`_renderPaperMode`, `_renderGlobalMode`, `_renderCollectionMode`, settings tab) still functional +- [ ] **CSS regression:** Deep-reading CSS doesn't break global/paper/collection dashboard layout +- [ ] **Empty state:** Fresh vault, no papers synced — verify deep-reading mode accessible or correctly gated + +--- + +## Recovery Strategies + +| Pitfall | Recovery Cost | Recovery Steps | +|---------|---------------|----------------| +| Mode ordering regression (deep-reading → paper) | LOW | Reorder mode detection: check `basename === 'deep-reading'` first. Single-function fix. | +| `discussion.json` GBK encoding corruption | MEDIUM | Re-run `/pf-paper` to regenerate with explicit UTF-8. Fix the Python writer. Existing files: detect via hex inspection and re-encode. | +| `discussion.json` schema version missing | HIGH | Write migration script: detect bare array format, wrap in envelope with `schema_version: "1"`. Must preserve all existing data. | +| `btoa()` crash on Chinese paths | LOW | Replace with `Buffer.from(str).toString('base64')`. Single-line fix per occurrence. | +| Empty `discussion.json` crashes dashboard | LOW | Add file-existence and content-validity guards. Four `if` branches in loader function. | +| Version badge permanently `v—` | LOW | Add fast synchronous version file cache; add `paperforge --version` fallback. | +| `active-leaf-change` mode oscillation | MEDIUM | Extract `_resolveModeForFile()`, add file identity guard, adjust debounce timing. Requires careful testing with split panes. | +| CSS namespace collision | MEDIUM | Add mode-specific wrapper class; audit CSS diff. Roll back CSS section if conflict found. | +| Jump button broken on missing file | LOW | Add `getAbstractFileByPath` existence check before `openLinkText`. | + +--- + +## Pitfall-to-Phase Mapping + +| Pitfall | Prevention Phase | Verification | +|---------|------------------|--------------| +| Mode detection ordering (P1) | Phase 1 — Deep-reading dashboard mode | Open deep-reading.md; verify mode badge shows "Deep Reading" | +| `fs.readFileSync` bypasses vault API (P2) | Phase 2 — AI discussion recorder | Check file read method in code review; verify modify events trigger refresh | +| `discussion.json` schema version missing (P3) | Phase 2 — AI discussion recorder | Inspect generated discussion.json for `schema_version: "1"` | +| `btoa()` on Chinese (P4) | Phase 2 — AI discussion recorder | Grep plugin JS for `btoa(`; test with Chinese-titled papers | +| `active-leaf-change` double-fire (P5) | Phase 1 — Deep-reading dashboard mode | Click between formal note and deep-reading.md in split panes | +| Empty `discussion.json` states (P6) | Phase 1 + Phase 2 | Test fresh paper with no AI discussions; verify empty state renders | +| Jump button assumes file exists (P7) | Phase 1 — Deep-reading dashboard mode | Click button on paper before running /pf-deep | +| Version badge race (P8) | Phase 4 — Bug fixes | Load plugin with no paperforge CLI; verify graceful fallback | +| "ai" row removal scope (P9) | Phase 4 — Bug fixes | Verify all render paths after removal; check Copy Context still works | +| `.md` vs `.json` inconsistency (P10) | Phase 2 — AI discussion recorder | Run doctor after generating discussions; compare .md and .json | + +--- ## Sources -- `.planning/PROJECT.md` — current v1.6 goals, constraints, and explicit anti-goals. Confidence: HIGH. -- `docs/ARCHITECTURE.md` — worker/agent split, local-first file boundaries, command centralization. Confidence: HIGH. -- `.planning/phases/15-deep-reading-queue-merge/15-LEARNINGS.md` — prior move toward canonical pure data acquisition functions. Confidence: HIGH. -- `.planning/phases/20-plugin-settings-shell-persistence/20-LEARNINGS.md` — proven risk of tracking/implementation drift and plugin thin-shell lessons. Confidence: HIGH. -- Context7 Obsidian developer docs: Plugin settings persistence and ItemView/View registration patterns. Confidence: HIGH. - - https://github.com/obsidianmd/obsidian-developer-docs/blob/main/en/Plugins/User%20interface/Settings.md - - https://github.com/obsidianmd/obsidian-developer-docs/blob/main/en/Plugins/User%20interface/Views.md -- Ink & Switch, “Local-first software: You own your data, in spite of the cloud” — local-first principle that local data is primary and derived surfaces should preserve ownership and rebuildability. Confidence: HIGH. - - https://www.inkandswitch.com/local-first/ -- Exa-discovered AI knowledge/RAG articles used only for product-shaping heuristics around explainability, retrieval governance, and avoiding hardcoded schemas. Confidence: LOW. - - https://redwerk.com/blog/rag-best-practices/ - - https://oleno.ai/blog/design-a-knowledge-base-that-makes-ai-content-writing-accurate-and-repeatable/ +- **Existing codebase analysis:** `paperforge/plugin/main.js` (lines 718-764, mode detection; lines 1246-1264, event subscriptions; lines 322, 428, `fs.readFileSync` patterns). Confidence: HIGH. +- **Obsidian Developer Docs:** Event system, `registerEvent()`, `active-leaf-change`, `debounce()` best practices. Context7 library `/obsidianmd/obsidian-developer-docs`. Confidence: HIGH. +- **Obsidian Forum #31841:** `active-leaf-change` event fires twice during tab switches — confirmed by multiple plugin developers. Confidence: HIGH. +- **Obsidian Forum #91927:** GB2312 to UTF-8 conversion corrupting Chinese files permanently — encoding mismatch between system code page and expected encoding. Confidence: HIGH. +- **opencode-obsidian Issue #28:** `btoa()` crash with Chinese characters in paths — `Buffer.from(str).toString('base64')` is the standard fix. Confidence: HIGH. +- **nodejs/undici Issue #5002:** Multi-byte UTF-8 character corruption at chunk boundaries when `setEncoding('utf8')` is used — confirmed in production with CJK text. Confidence: HIGH. +- **paperclipai/paperclip Issue #3940:** GBK-UTF8 encoding mismatch on Windows with CJK child process output — root cause is `iconv-lite` corruption + missing `PYTHONIOENCODING` env var. Confidence: MEDIUM (fix validated, specific library version dependent). +- **Excalidraw plugin commit 5c628e0:** Real-world example of debounce timer cleanup and observer disposal in Obsidian plugin `onunload`. Confidence: HIGH. +- **obsidian-current-view commit ad110f7:** Real-world example of replacing `requestAnimationFrame` with `setTimeout` debounce and building lookup maps once per call. Confidence: HIGH. +- **Templater Issue #1629:** `app.vault.modify()` race condition when multiple handlers modify the same file — relevant for discussion.json concurrent access. Confidence: MEDIUM. +- **Project architecture context:** `.planning/research/ARCHITECTURE.md` — thin-shell plugin principle, canonical index as read model, file ownership boundaries. Confidence: HIGH. +- **Project defect history:** `.planning/research/DEFECTS.md` — encoding issues, path resolution brittleness, worker/plugin boundary problems. Confidence: HIGH. + +--- + +*Pitfalls research for: PaperForge v1.8 — AI Discussion & Deep-Reading Dashboard* +*Researched: 2026-05-06* +*Research mode: Project Research — PITFALLS* diff --git a/.planning/research/STACK.md b/.planning/research/STACK.md index 9acb64fa..c325c134 100644 --- a/.planning/research/STACK.md +++ b/.planning/research/STACK.md @@ -1,201 +1,218 @@ -# Technology Stack +# Stack Research — v1.8 AI Discussion Recording & Deep-Reading Dashboard -**Project:** PaperForge v1.6 literature asset foundation -**Researched:** 2026-05-03 +**Domain:** AI discussion recording + deep-reading dashboard mode for PaperForge Obsidian plugin +**Researched:** 2026-05-06 +**Confidence:** HIGH + +## Executive Summary + +v1.8 adds two new capabilities to the existing PaperForge stack: (1) AI discussion recording that captures `/pf-paper` and agent chat interactions into structured `ai/` records, and (2) a 4th dashboard mode activated when the user views `deep-reading.md`. Both features extend the existing thin-shell plugin architecture — the plugin reads JSON from the filesystem and renders Pure CSS/DOM components; the Python side provides templates and optional recording helpers. **No new npm dependencies, no new Python packages.** All additions are new modules/files within the existing architecture. ## Recommended Stack -This milestone should stay **Python-first for business logic** and **CommonJS Obsidian-shell only for UI**. The plugin should render the canonical index and invoke CLI commands; it should not recompute lifecycle, health, maturity, or AI-context state on the JavaScript side. +### Core Technologies (unchanged from v1.6–v1.7) -### Core Framework | Technology | Version | Purpose | Why | |------------|---------|---------|-----| -| Python | 3.10+ (existing) | Single owner of config, lifecycle, health, maturity, context-pack generation | Already owns workers/CLI and is the only place that can safely unify business rules without duplicating them in the plugin | -| Pydantic | 2.13.3 | Typed models for `paperforge.json`, canonical asset index, health records, maturity scoring payloads, AI context manifests | Best fit for strict runtime validation plus `model_json_schema()` generation from one source of truth; keeps Python authoritative and produces machine-readable contracts for the plugin/tests | -| Obsidian plugin (`main.js`, CommonJS) | existing thin shell | Dashboard/settings/commands only | Current plugin already shells out to `python -m paperforge status --json`; extend that pattern instead of adding a second domain layer | +| Obsidian CommonJS plugin | existing (`main.js`) | Dashboard/settings/commands UI | Pure Obsidian API — no bundler, no npm deps, no build step | +| Python 3.10+ | existing | Single owner of business logic, templates, schema | Already owns config, lifecycle, health, maturity, index; extends naturally to discussion templates | +| Filesystem JSON | UTF-8, no schema library needed | `discussion.json` per paper | Plugin reads via `fs.readFileSync`, localStorage-free, vault-native, traceable | +| Filesystem Markdown | Obsidian-flavored | `discussion.md` per paper | Human-readable, Obsidian-editable, wikilink-compatible | -### Database -| Technology | Version | Purpose | Why | -|------------|---------|---------|-----| -| Filesystem JSON snapshot | UTF-8 JSON, schema-versioned | Canonical derived literature asset index | Local-first, Git/backup friendly, easy for Python to emit and plugin to read, no daemon/service required | -| Optional JSONL append log | UTF-8 JSON Lines | Append-only lifecycle/diagnostic history if needed later | Good long-lived audit primitive without committing to SQLite/event sourcing in v1.6 | +### File Format Additions (NEW for v1.8) -### Infrastructure -| Technology | Version | Purpose | Why | -|------------|---------|---------|-----| -| filelock | 3.29.0 | Cross-process locking around index/context-pack writes | Verified cross-platform, and on Windows uses OS-level locking; prevents sync/OCR/plugin-triggered commands from corrupting shared JSON files | -| `tempfile` + `os.replace` | stdlib | Atomic file writes | Windows-safe write pattern for canonical JSON artifacts; avoids partial writes when commands are interrupted | -| `pathlib`, `hashlib`, `json` | stdlib | Path normalization, content fingerprints, serialization | Already aligned with current codebase and enough for local asset indexing | +#### 1. `discussion.json` — Structured AI Q&A Record -### Supporting Libraries -| Library | Version | Purpose | When to Use | -|---------|---------|---------|-------------| -| jsonschema | 4.26.0 | Validate emitted JSON artifacts against generated schemas in tests and `doctor`-style diagnostics | Use in Python tests/tooling, not as plugin runtime logic | +**Location:** `<paper_workspace>/ai/discussion.json` -## File-Format Standards - -### 1. `paperforge.json` stays the human-edited source of truth -- Keep it as the canonical config file. -- Add `schema_version`. -- Validate it in Python with a `PaperForgeConfig` Pydantic model. -- Keep secrets out of it; API keys remain in `.env` / environment. - -Recommended structure for new v1.6 fields: +**Schema (v1):** ```json { "schema_version": "1", - "vault_config": { ...existing paths... }, - "index": { - "output": "<system_dir>/PaperForge/indexes/library-assets.v1.json" - }, - "health": { - "enable_maturity_scoring": true - }, - "context": { - "output_dir": "<system_dir>/PaperForge/context", - "default_max_chars": 24000 + "paper_key": "ABCDEFG", + "generated_at": "2026-05-06T12:00:00+08:00", + "source": "/pf-paper", // or "/pf-deep", "agent-chat" + "history": [ + { + "index": 1, + "timestamp": "2026-05-06T12:00:00+08:00", + "question": "这篇论文的主要发现是什么？", + "answer": "该研究发现...", + "tags": ["background", "results"], + "agent_model": "deepseek-v4-pro" + } + ], + "summary": { + "total_qa": 5, + "last_updated": "2026-05-06T12:30:00+08:00", + "top_tags": ["methods", "results"] } } ``` -### 2. Canonical asset index = one derived snapshot file -Recommended path: +**Why this shape:** +- `schema_version` for forward compatibility +- `history[]` is a flat append-only list (no nested threads — keeps dashboard rendering simple) +- `index` field enables stable references even if history is reordered +- `tags` enable future dashboard filtering without schema change +- `summary` provides pre-computed metadata the plugin can read with a single parse (avoids looping through all history entries for total count) -`<system_dir>/PaperForge/indexes/library-assets.v1.json` +**Integration:** Plugin reads this file directly via `fs.readFileSync` when rendering the deep-reading dashboard. No Python intermediary needed for read path. -Recommended top-level shape: +#### 2. `discussion.md` — Human-Readable Q&A Log -```json -{ - "schema_version": "1", - "generated_at": "2026-05-03T12:00:00Z", - "paperforge_version": "1.6.x", - "vault": { "...": "..." }, - "summary": { "...": "..." }, - "assets": [ - { - "zotero_key": "ABCDEFG", - "intent": { "analyze": true, "do_ocr": true }, - "artifacts": { "pdf": {...}, "ocr": {...}, "formal_note": {...}, "figures": {...} }, - "derived": { - "lifecycle_state": "fulltext_ready", - "health": { "library": "healthy", "pdf": "healthy", "ocr": "warning" }, - "maturity": { "level": 3, "score": 0.72, "next_steps": ["run_deep_reading"] }, - "ai_ready": true - } +**Location:** `<paper_workspace>/ai/discussion.md` + +**Template format:** + +```markdown +# AI 讨论记录 — 文献标题 + +> **来源:** /pf-paper | **生成时间:** 2026-05-06 12:00 | **模型:** deepseek-v4-pro + +--- + +## 讨论 1 + +**问题:** 这篇论文的主要发现是什么？ + +**解答:** 该研究发现... + +*标签: `background`, `results`* + +--- + +## 讨论 2 + +**问题:** 研究者用了什么统计方法？ + +**解答:** ... + +*标签: `methods`* +``` + +**Why this format:** +- Each discussion is a `## 讨论 N` section — natural Obsidian heading hierarchy +- `**问题:**` and `**解答:**` bold markers are scannable in both Reading and Source mode +- Horizontal rules (`---`) visually separate discussions +- Tags as inline code spans — searchable in Obsidian's global search +- Frontmatter-compatible metadata line at top + +### Plugin JS Additions (NEW for v1.8 — no new npm deps) + +All additions stay within the single `paperforge/plugin/main.js` file and `paperforge/plugin/styles.css`. + +| Module/Function | Type | Purpose | +|---------|------|---------| +| `_detectAndSwitch()` extension | Mode detection logic | Add `deep-reading` as 4th mode: detect when active file path ends with `deep-reading.md` and resolve paper context from workspace path | +| `_renderDeepReadingMode()` | New render function | Renders deep-reading dashboard: status bar, Pass 1 summary, AI discussion history | +| `_renderDiscussionHistory()` | New render function | Reads `discussion.json` from paper `ai/` directory and renders Q&A cards | +| `_readDiscussionJson(key)` | New utility | Resolves `ai_path` → reads `discussion.json` → returns parsed object or null | +| `_renderPass1Summary()` | New render function | Renders Pass 1 overview extracted from deep-reading.md content | +| `_renderPaperMode()` extension | Existing function | Add "Jump to Deep Reading" contextual button when `deep_reading_path` exists | +| `_versionBadge` fix | Bug fix | Restore version display by reading from `paperforge/__init__.py` `__version__` or `manifest.json` | +| "ai" row removal | Bug fix | Remove the meaningless "ai" UI row (track down which render path creates it) | + +**Mode detection logic (how `_detectAndSwitch()` grows):** + +```javascript +// Existing: checks for .base, .md with zotero_key +// NEW: check if active file basename is "deep-reading.md" +if (ext === 'md' && activeFile.basename === 'deep-reading') { + // Resolve paper key from parent directory name + const parentDir = activeFile.parent.path; // e.g., "Literature/骨科/ABC12345 - Title" + const match = parentDir.match(/([A-Z0-9]{8})/); + if (match) { + this._currentPaperKey = match[1]; + this._currentPaperEntry = this._findEntry(match[1]); + this._switchMode('deep-reading'); + return; } - ] } ``` -Rules: -- `library-record` frontmatter remains **user intent**. -- `meta.json`, formal notes, OCR files, figure-map stay **machine artifacts**. -- `library-assets.v1.json` is **derived state only**. -- Plugin reads this file; it does not infer the state itself. +### CSS Additions (NEW for v1.8) -### 3. Generated schema file -Recommended path: +All additions go into the existing `paperforge/plugin/styles.css` file after Section 17. -`<system_dir>/PaperForge/indexes/library-assets.schema.json` +| CSS Class | Purpose | +|-----------|---------| +| `.paperforge-deep-reading-view` | Root container for deep-reading mode layout (flex column, gap: 20px) | +| `.paperforge-dr-status-bar` | Status indicator bar: shows Pass 1/2/3 completion, OCR status badge, last-updated timestamp | +| `.paperforge-dr-pass-summary` | Pass 1 summary card: bordered card with muted background for the overview text | +| `.paperforge-dr-discussion-card` | Individual Q&A card: question in bold, answer below, tag chips, timestamp | +| `.paperforge-dr-discussion-list` | Scrollable container for stacked discussion cards | +| `.paperforge-dr-tag-chip` | Small pill for tags (font-size: 10px, border-radius: 8px) | +| `.paperforge-dr-empty` | Empty state: "No AI discussions recorded yet" with muted styling | +| `.paperforge-dr-section-title` | Section header within deep-reading view (uppercase, letter-spaced) | -Generate from Pydantic via `model_json_schema()`. This gives: -- a contract for future migrations, -- validation in tests/doctor, -- a stable interface for the plugin without duplicating Python rules. +### Python Additions (NEW for v1.8 — no new dependencies) -### 4. AI context pack format -Use a folder, not a database blob: +| File | Purpose | +|------|---------| +| `paperforge/worker/discussion.py` | NEW module: `record_discussion(key, vault, question, answer, source, model)` — writes both `discussion.md` and `discussion.json` to paper `ai/` directory | +| `paperforge/worker/discussion.py` | `load_discussion_json(key, vault)` → dict — reads existing discussion.json, returns parsed dict | +| `paperforge/worker/discussion.py` | `append_discussion(key, vault, qa_pair)` — appends to existing discussion history (read-modify-write atomic via tempfile + os.replace) | -`<system_dir>/PaperForge/context/<scope>/<id>/` +**Why a separate module:** +- Discussion recording crosses the Worker/Agent boundary (Agent generates content, Worker writes files) +- Keeps discussion I/O isolated from OCR, sync, and index logic +- Enables future `/pf-discuss` command or discussion-search features without refactoring -Contents: -- `manifest.json` — typed metadata, provenance, hashes, included files -- `context.md` — user/LLM-ready packaged text +**No new Python packages required.** All I/O uses `json`, `pathlib`, and `datetime` (stdlib). Atomic write uses the same `tempfile` + `os.replace` pattern already proven in `asset_index.py`. -This is Obsidian-compatible, inspectable, and easy to copy/share locally. +### Integration Points (with existing canonical index) -## Implementation Primitives - -1. **Python domain models** - - `PaperForgeConfig` - - `AssetIndex` - - `AssetRecord` - - `AssetHealth` - - `LifecycleState` - - `MaturityReport` - - `ContextPackManifest` - -2. **One index builder module** - - Example boundary: `paperforge/indexing/` - - Reads library-records, OCR `meta.json`, formal notes, figure-map, exports - - Produces the canonical index snapshot - -3. **One health engine** - - Example boundary: `paperforge/health/` - - Pure functions: PDF health, OCR health, Base/template health, overall library health - - Reused by `status`, `doctor`, plugin dashboard, context pack generation - -4. **One context-pack generator** - - Example boundary: `paperforge/context/` - - Builds `ask-this-paper`, `ask-this-collection`, `copy-context-pack` payloads from canonical index + source artifacts - -5. **One write discipline** - - `FileLock(...).acquire()` around derived-artifact writes - - write temp file - - `os.replace()` into final path - -6. **One plugin integration contract** - - Plugin invokes CLI commands such as `paperforge index refresh`, `paperforge status --json`, `paperforge context pack ...` - - Plugin reads `library-assets.v1.json` - - Plugin never re-implements scoring/health/lifecycle logic - -## What Should Stay in Stdlib / Current Stack - -| Keep | Why | -|------|-----| -| `json` | Canonical index is not large enough to justify a binary or DB format yet | -| `pathlib` | Existing code is already path-centric and Windows-aware | -| `hashlib` | Enough for PDF/fulltext/context-pack fingerprints and stale-cache detection | -| Existing CLI/worker module pattern | Already proven in v1.2-v1.5 and matches the thin-shell plugin goal | -| Existing Obsidian CommonJS plugin style | No build-system migration needed for this milestone | +| Integration | Direction | How | +|-------------|-----------|-----| +| `ai_path` in canonical index | Read by plugin | Already exists: `entry.ai_path` = `"Literature/<domain>/<key> - <Title>/ai/"` | +| Resolve `discussion.json` path | Plugin computed | `path.join(vaultPath, entry.ai_path, 'discussion.json')` | +| Deep-reading mode detection | Plugin computed | Check `activeFile.basename === 'deep-reading'` + resolve key from parent directory | +| Jump-to-deep-reading button visibility | Plugin check | `entry.deep_reading_path` non-empty → show button | +| Discussion recording write path | Python writes | `record_discussion()` resolves paths via `paperforge_paths()` → writes to workspace ai/ | ## What NOT to Add -| Category | Recommended | Alternative | Why Not | -|----------|-------------|-------------|---------| -| Index storage | JSON snapshot + optional JSONL history | SQLite | Adds migration/query complexity and hides state from the vault filesystem without solving a proven scale bottleneck | -| Config typing | Pydantic model on top of current loader | `pydantic-settings` | Helpful for env-heavy apps, but PaperForge's real source of truth is `paperforge.json`; adding another settings layer is unnecessary in v1.6 | -| Plugin validation | Python-generated schema + version check | AJV / Zod in plugin | Creates a second contract surface in JS and pushes business rules toward the plugin | -| File watching | Explicit refresh after worker commands | watchdog / daemon | Violates the local-simple architecture and is unnecessary for user-triggered workflows | -| Search/index engine | Current filesystem artifacts + canonical JSON | Elasticsearch / LanceDB / vector DB | Overkill for milestone scope; AI packaging needs traceable bundles first, not semantic infra | -| YAML parser | Current narrow frontmatter handling + Python indexer | PyYAML / round-trip YAML libs | Extra dependency and formatting churn risk; users do not need full YAML mutation for v1.6 | +| Category | Avoid | Why | Use Instead | +|----------|-------|-----|-------------| +| JS dependencies | Any npm package (React, Vue, chart libs, date-fns) | Plugin must stay pure Obsidian CommonJS; adding a build chain breaks the single-file deployment model and Obsidian Community Plugin requirements | Pure DOM API (`createEl`, `addClass`, `setText`) | +| Database for discussions | SQLite, IndexedDB, localStorage | Discussions belong to the paper workspace — they must be vault-native, backup-friendly, and git-trackable | Filesystem JSON + Markdown in `ai/` | +| Python dependencies for discussion | `pydantic`, `jsonschema` | Overkill for a flat Q&A list; adds import cost and version coupling for a simple structure | Plain `dict` with documented keys, validated by `assert` in tests | +| Second plugin file | Splitting `main.js` into modules | Adds complexity without benefit at ~2100 lines; Obsidian plugin import resolution is fragile | Inline functions in `main.js` (as all existing render functions are) | +| Schema library in plugin | AJV, Zod, or custom JSON validator | Plugin should read, not validate; schema enforcement lives in Python tests | Trust but handle: `try/catch` around `JSON.parse`, fallback to empty state | +| New CLI command (v1.8) | `paperforge discuss` or `paperforge record` | Discussion recording happens inside Agent sessions; adding a CLI command would require users to manually run it after each Q&A | Python module callable from Agent scripts (`/pf-paper` → calls `record_discussion()` internally) | + +## Version Compatibility + +| Component | Current Version | Notes | +|-----------|----------------|-------| +| PaperForge Python | 1.4.15 (→ 1.8.0) | No breaking changes to existing CLI or index schema | +| Canonical index schema | v2 (`formal-library.json`) | `ai_path` field already present; no schema version bump needed | +| Obsidian API | 1.5+ | Existing API surface used (ItemView, metadataCache, vault.adapter) | +| Node.js (for Obsidian) | 18+ (Electron embedded) | `fs.readFileSync`, `path.join` — all stdlib since Node 0.x | ## Installation +No new installation steps. v1.8 is additive: + ```bash -# Core additions -pip install "pydantic>=2.13,<3" "filelock>=3.29,<4" +# No new pip packages needed +# No new npm packages needed -# Dev / contract validation -pip install -D "jsonschema>=4.26,<5" +# Plugin update: replace main.js + styles.css + manifest.json + versions.json +# Python update: add paperforge/worker/discussion.py ``` -## Integration Notes for Existing Architecture - -- Extend `paperforge.config` to return validated Pydantic-backed config objects, but keep current precedence semantics. -- Add a dedicated `index refresh` command that all relevant workers can call after successful mutations. -- Refactor `status --json` to consume the canonical index instead of recounting raw files independently. -- Have the plugin dashboard read index summary fields first, and shell out only for refresh/actions. -- Keep OCR/fulltext/figure outputs where they already live; the new index should reference them, not relocate them. - ## Sources -- Pydantic docs via Context7 — typed models, validation, `model_dump()`, `model_json_schema()` — HIGH confidence — https://context7.com/pydantic/pydantic -- Pydantic PyPI JSON — current version `2.13.3` — HIGH confidence — https://pypi.org/pypi/pydantic/json -- filelock docs via Context7 — platform-aware locking, Windows OS-level locking — HIGH confidence — https://context7.com/tox-dev/filelock -- filelock PyPI JSON — current version `3.29.0` — HIGH confidence — https://pypi.org/pypi/filelock/json -- jsonschema docs via Context7 — Draft 2020-12 validator support — HIGH confidence — https://context7.com/python-jsonschema/jsonschema/v4.25.1 -- jsonschema PyPI JSON — current version `4.26.0` — HIGH confidence — https://pypi.org/pypi/jsonschema/json -- Existing repo context: `pyproject.toml`, `paperforge/config.py`, `paperforge/worker/status.py`, `paperforge/plugin/main.js` — HIGH confidence +- Existing codebase analysis: `paperforge/plugin/main.js` (2067 lines), `paperforge/plugin/styles.css` (1325 lines), `paperforge/worker/asset_index.py` (577 lines), `paperforge/worker/asset_state.py` (243 lines), `paperforge/worker/sync.py` (1829 lines) — HIGH confidence +- `.planning/PROJECT.md` — v1.8 milestone definition, ai/ directory already created per paper — HIGH confidence +- AGENTS.md — Lite architecture, Worker/Agent split, plugin thin-shell constraint — HIGH confidence +- `.planning/research/STACK.md` (v1.6) — existing stack decisions, Pydantic version, filelock version, "what NOT to add" — HIGH confidence +- Actual test fixtures: `tests/test_asset_state.py` — ai_path field verified in lifecycle/health computation — HIGH confidence + +--- + +*Stack research for: PaperForge v1.8 AI Discussion Recording & Deep-Reading Dashboard* +*Researched: 2026-05-06* diff --git a/.planning/research/SUMMARY.md b/.planning/research/SUMMARY.md index 3daa06da..650904d6 100644 --- a/.planning/research/SUMMARY.md +++ b/.planning/research/SUMMARY.md @@ -1,194 +1,353 @@ # Project Research Summary -**Project:** PaperForge v1.6 literature asset foundation -**Domain:** 基于 Zotero + Obsidian 的本地优先文献资产管理与 AI 上下文基础设施 -**Researched:** 2026-05-03 +**Project:** PaperForge v1.8 — AI Discussion Recording & Deep-Reading Dashboard +**Domain:** Brownfield Obsidian plugin — extending mode-based dashboard with AI discussion capture +**Researched:** 2026-05-06 **Confidence:** HIGH ## Executive Summary -PaperForge v1.6 不应被做成“又一个提示词按钮集合”，而应被收敛为一个建立在现有 Worker/Agent 双层架构之上的“文献资产底座”。研究结论非常一致：Zotero 继续做书目与附件真相源，Obsidian 继续做知识工作台，Python 继续做业务语义与派生状态的唯一所有者；新增能力的核心不是更多提取功能，而是把每篇文献当前“有什么、缺什么、下一步做什么”稳定地表示出来。 +PaperForge v1.8 adds two tightly-coupled capabilities to the existing PaperForge Obsidian plugin: **(1) AI discussion recording** that captures `/pf-paper` and `/pf-deep` conversations into structured `ai/discussion.json` and human-readable `ai/discussion.md` files within each paper's workspace directory; and **(2) a 4th dashboard mode** (`deep-reading`) activated automatically when the user opens a `deep-reading.md` file, showing Pass 1/2/3 completion status, Pass 1 summary, and recent AI Q&A history. -推荐路径是：围绕现有 `paperforge.json`、`library-records`、OCR 产物、正式文献笔记，演进出一个由 Python 统一生成的规范资产索引，并让 plugin 只做 thin shell 展示与命令触发。这样可以把 lifecycle、health、maturity、AI-ready 等判断从当前分散的命令/界面里收敛到一个可重建的派生读模型中，再在此基础上提供 ask-this-paper、ask-this-collection、copy-context-pack 等通用 AI 入口。 +The recommended approach is **thin-shell extension** — zero new dependencies, zero new CLI commands. The Obsidian plugin (CommonJS, single `main.js` file) gains one new mode renderer and one mode detection branch. A new Python module (`paperforge/worker/discussion.py`) handles writing discussion files at the end of Agent sessions. The canonical index (`formal-library.json`) serves as the bridge — JS reads pre-computed lifecycle/health/maturity from it, while `discussion.json` (vault-internal) should use `vault.adapter.read()` for cache consistency, **not** `fs.readFileSync()` as the older index pattern does. -最高风险不在“技术做不做得出”，而在 brownfield 演进中制造多重真相源、把 canonical index 用成手工数据库、以及在 plugin 中重复实现 Python 业务规则。v1.6 的里程碑设计必须先锁定字段归属、重建契约与迁移/回滚路径，再做 dashboard、成熟度评分和 AI context packaging；否则表面功能越多，状态漂移和维护成本只会越严重。 +The three key risks are: (1) mode detection ordering — `deep-reading.md` carries the same `zotero_key` frontmatter as the formal note and will incorrectly trigger per-paper mode unless the `deep-reading.md` filename check comes *first*; (2) encoding corruption on Windows with CJK locales — Chinese Q&A content written by Python must use explicit `encoding='utf-8'` and Node.js must read with matching encoding to avoid mojibake; and (3) `discussion.json` must include `schema_version` from day one to prevent silent data loss on format changes. All three are preventable with the right discipline in Phase 1 and Phase 2. ## Key Findings ### Recommended Stack -v1.6 的推荐栈是“Python-first + CommonJS thin-shell plugin”，不是引入新前端层，也不是把业务逻辑迁到 JS。核心新增依赖很克制：用 Pydantic 建立 `paperforge.json`、canonical index、health/maturity/context manifest 的强类型契约；用 filelock 与 `tempfile` + `os.replace()` 保证 Windows 友好的跨进程锁与原子写入；用 jsonschema 做测试和 `doctor` 级别的契约校验。 +v1.8 is purely additive to the existing v1.6–v1.7 stack. **No new npm packages, no new Python packages.** The plugin remains a single CommonJS file (`main.js`, currently ~2067 lines) with CSS in `styles.css` (~1325 lines). Python gains one new module (`discussion.py`) using only stdlib (`json`, `pathlib`, `datetime`, `tempfile`, `os`). -最重要的栈结论不是“加什么”，而是“不要加什么”：不要上 SQLite、不要上 watchdog/daemon、不要在 plugin 里引入 AJV/Zod 作为第二套契约、不要为 v1.6 引入向量库/搜索引擎。当前 PaperForge 的问题是资产状态不可统一解释，不是检索基础设施不足。 +**Core technologies (unchanged):** +- **Obsidian CommonJS plugin** (`main.js`, no bundler): Dashboard/settings/commands UI — pure Obsidian API, no build step +- **Python 3.10+** (existing CLI): Single owner of business logic, templates, schema; extends naturally to discussion recording +- **Filesystem JSON** (`formal-library.json`, `discussion.json`): Contract between Python and JS; vault-native, git-trackable, backup-friendly +- **Filesystem Markdown** (`discussion.md`): Human-readable companion, Obsidian-editable, wikilink-compatible -**Core technologies:** -- **Python 3.10+**：继续作为 config、lifecycle、health、maturity、context-pack 的唯一业务所有者，避免 JS/Python 双实现漂移。 -- **Pydantic 2.13.3**：为 `paperforge.json`、`formal-library.json` 演进后的 envelope、context pack manifest 提供单一类型真相与 schema 输出。 -- **filelock 3.29.0**：为 index/context pack 写入提供跨进程锁，防止 sync、ocr、plugin 同时写文件造成损坏。 -- **`tempfile` + `os.replace()`**：保证 canonical JSON/缓存产物原子落盘，避免中断时出现半写文件。 -- **CommonJS Obsidian plugin（现有）**：仅负责 dashboard、settings、命令触发与结果展示，不承担状态推导。 -- **jsonschema 4.26.0（测试/诊断）**：用于验证生成的 schema 与导出产物，提升 doctor/CI 的可验证性。 +**New file formats for v1.8:** +- `ai/discussion.json` — structured AI Q&A record with `schema_version`, `paper_key`, `sessions[]` array containing `session_id`, `agent`, `started`, and `qa_pairs[]` +- `ai/discussion.md` — human-readable Q&A log with session-grouped `##` headings, `**问题:**`/`**解答:**` markers, timestamp metadata -**关键版本/格式要求：** -- `paperforge.json` 增加 `schema_version` -- 继续以文件系统 JSON snapshot 为主，不引入数据库 -- canonical index 采用版本化 envelope，而不是裸列表 +**Key "what NOT to add" decisions (all researchers agree):** +- No npm packages (React, Vue, chart libs) — breaks single-file deployment +- No database (SQLite, IndexedDB) — discussions belong to the paper workspace, must be vault-native +- No Python schema libraries (Pydantic, jsonschema) — overkill for flat Q&A list +- No second plugin file — import resolution is fragile in Obsidian +- No new CLI command (`paperforge discuss`) — recording happens inside Agent sessions ### Expected Features -研究对“必须做什么”和“不要做什么”划分很清楚。v1.6 的 table stakes 不是炫技式 AI，而是把文献库做成一个可读、可修、可重建、可复用的资产系统；真正的 differentiator 也不是医学专用提取表，而是可追溯的 context pack、统一 health/lifecycle 语义与可执行的 next-step guidance。 +**Must have (table stakes — v1.8 launch):** +- AI Discussion Recorder: Writes `discussion.md` + `discussion.json` when `/pf-paper` or `/pf-deep` completes — researchers expect past AI conversations next to the paper, not scattered in browser tabs +- Deep-Reading Dashboard Mode: Plugin detects `deep-reading.md` as active file, switches to dedicated mode showing Pass 1/2/3 status + AI Q&A history — users want at-a-glance reading progress without scrolling +- Jump-to-Deep-Reading Button: Contextual button on per-paper dashboard card that opens deep-reading.md — bridges paper lifecycle view to reading content +- Version Number Display Fix: Restore the `_versionBadge` — users need to confirm version for issue reporting +- "ai" UI Row Removal: Remove the meaningless row — erodes trust in dashboard accuracy -**Must have（table stakes）:** -- **Canonical asset index**：统一回答每篇文献有哪些资产、缺哪些资产、当前可用于什么。 -- **显式 lifecycle state model**：区分 imported、indexed、pdf_ready、fulltext_ready、deep_read_done、ai_context_ready 等派生状态。 -- **Library health surfaces**：能看到 PDF、路径、OCR、note/template/base 的健康状况，并支持聚合到 collection/library。 -- **Stable per-asset schema**：统一标识符、路径、provenance、readiness 字段，保证长期可维护。 -- **Derived queue / next-step views**：从派生状态给出“下一步该 sync / ocr / repair / pf-deep 什么”，而不是只暴露原始 frontmatter。 -- **Idempotent rebuild / refresh**：任何修复后都能安全重建索引和视图，不污染正式笔记。 +**Should have (v1.8.x follow-up, deferrable):** +- Discussion session merging (append, not overwrite, on repeated `/pf-paper` for same paper) +- Session metadata in discussion.json (model, duration, agent type) +- Pass completion percentage calculation -**Should have（differentiators）:** -- **Ask-this-paper context pack**：把单篇论文打包为可追溯 AI 输入，包含 metadata、fulltext、figures、note links、provenance。 -- **Ask-this-collection / copy-context-pack**：把 collection 级资产打包给 NotebookLM 风格、但本地优先的综合工作流。 -- **Maturity / workflow level scoring**：用透明、可解释的等级/评分告诉用户这篇论文距离“AI-ready”还有多远。 -- **Actionable diagnostics with fix paths**：不仅报错，还明确推荐 `sync`、`ocr`、`repair`、重建 note 或重建 index。 -- **Thin-shell plugin dashboard**：基于 canonical index 的产品化界面，而不是第二套业务引擎。 +**Future consideration (v2+):** +- Cross-library discussion search (rely on Obsidian built-in search) +- Discussion export to AI context packs +- Deep-reading maturity integration into maturity gauge -**Defer（v2+ 或保持为可选能力）:** -- 学科专用 extraction outputs（PICO、机制表、参数表等） -- 自动从 worker 触发 deep-reading agent -- 大量 prompt-specific 按钮与“功能爆炸”式 AI surface -- 替代 Zotero 的参考文献管理能力 -- 云协作/远程同步 -- Litmaps/ResearchRabbit 风格的完整发现图谱产品 - -**Anti-features（本里程碑明确不应产品化）:** -- 不把每个成功 prompt 升级为核心功能 -- 不把 domain-specific extraction schema 烙进 core index -- 不在 plugin 里做第二套 lifecycle/health 逻辑 -- 不做黑箱式“AI 自动整理一切” +**Anti-features explicitly rejected:** +- Auto-recording all agent conversations (creates noise, violates worker/agent boundary) +- Full chat transcription in markdown (unstructured, duplicates agent's internal logs) +- Deep-reading dashboard as replacement for deep-reading.md (dashboard is index/summary view only) +- Real-time dashboard updates during deep-reading (agent executes outside plugin; refresh on active-leaf-change is sufficient) +- Discussion search across all papers (Obsidian's built-in search already handles this) ### Architecture Approach -架构结论非常明确：不要新造第二个 canonical index，也不要重写现有本地文件架构；应当直接演进现有 `<system_dir>/PaperForge/indexes/formal-library.json`，把它从“笔记列表”升级为“版本化 canonical derived read model”。它读取 `paperforge.json` 作为配置真相、`library-record` 作为用户意图真相、OCR 目录作为机器事实真相、正式笔记中的 `## 🔍 精读` 作为 deep-reading 真相，再由 Python 一次性推导 lifecycle/health/maturity/context readiness，供 CLI、plugin、Bases 复用。 +The architecture extends PaperForge's existing **thin-shell plugin pattern**: Python owns all business logic and writes structured data to the filesystem; the Obsidian JS plugin reads that data and renders Pure CSS/DOM components. No business logic is duplicated in JS. -**Major components:** -1. **配置真相层（`paperforge.json` / `paperforge.config`）** — 统一路径解析与运行时配置，plugin 只做镜像缓存，不做独立默认值真相源。 -2. **资产索引构建层（建议 `asset_index.py`）** — 汇总 library-record、OCR meta、formal note、figure-map 等，生成版本化 `formal-library.json`。 -3. **状态/健康推导层（建议 `asset_state.py`）** — 以纯函数形式统一 lifecycle、readiness、health、maturity 与 next-step 规则。 -4. **上下文打包层（建议 `context_pack.py`）** — 基于 canonical item 生成 per-paper / per-collection AI context packs。 -5. **thin-shell plugin + Base 镜像层** — 只读 canonical index 或 CLI JSON 输出，展示 dashboard、queue、settings 和执行入口。 +The key architectural addition is a **4th mode dispatch** in the existing `_detectAndSwitch()` → `_switchMode()` → `case 'mode': renderX()` pattern. The mode detection hierarchy (in strict order) becomes: -**架构立场（里程碑必须坚持）：** -- 演进 `formal-library.json`，不要新增并行索引文件 -- `library-record` 保存用户意图 + machine mirror，不承载所有真相 -- plugin 不决定 OCR 是否完成、paper 是否 AI-ready、health 是否红黄绿 -- repair 永远修 source artifacts，再重建 index，不直接修 index +``` +1. no active file → 'global' (existing) +2. .base file → 'collection' (existing) +3. deep-reading.md → 'deep-reading' (NEW — MUST precede zotero_key check) +4. .md with zotero_key → 'paper' (existing) +5. fallback → 'global' (existing) +``` + +**Major components affected:** +1. **`_detectAndSwitch()` + `_switchMode()`** — Mode detection extended with deep-reading.md filename check; must be checked *before* zotero_key frontmatter to prevent hijacking by per-paper mode +2. **`_renderDeepReadingMode()` (NEW)** — Dedicated render method consuming data from formal-library.json (lifecycle/health/maturity), deep-reading.md (Pass 1 summary), and ai/discussion.json (AI Q&A history) +3. **`_renderPaperMode()` extension** — Adds "Jump to Deep Reading" contextual button when `deep_reading_path` exists +4. **`discussion.py` (NEW Python module)** — `record_discussion()` writes both discussion.md and discussion.json; `load_discussion_json()` reads existing record; `append_discussion()` appends new Q&A pairs atomically +5. **`asset_index.py` build_envelope()** — Adds `paperforge_version` field for the version badge fix + +**Key data flow:** +``` +User runs /pf-deep → ld_deep.py generates scaffold → Agent completes session + → discussion_recorder.py writes ai/discussion.{md,json} +User opens deep-reading.md → _detectAndSwitch() detects it → _renderDeepReadingMode() + → reads formal-library.json (status badges) + deep-reading.md (Pass 1 summary) + ai/discussion.json (Q&A history) +``` ### Critical Pitfalls -1. **多重真相源并存** — 必须先发布字段归属矩阵：`paperforge.json` 管配置，plugin `data.json` 仅作 UI cache，`library-record` 管用户意图，OCR/meta 管机器事实，canonical index 只做派生投影。 -2. **把 canonical index 用成手工数据库** — index 必须可删除、可重建、可追溯；任何手工覆写都应放在独立 override 层，而不是直接改 index。 -3. **把意图、事实、派生 readiness 混成一个状态机** — `analyze/do_ocr`、OCR 完成、deep-reading 完成、AI-ready 必须分层表示；readiness 应计算得出，不应靠用户手改。 -4. **在 Obsidian plugin 里重写 health/lifecycle 逻辑** — 所有 dashboard 数字和 next-step 都应来自 Python 输出的 JSON 契约，否则 CLI/plugin 很快会分叉。 -5. **做出“看起来智能、实际上不可解释”的成熟度分数** — 若要评分，必须能拆成检查项、权重、证据与修复建议，优先做 level/band 而非伪精确百分制。 -6. **没有 brownfield 迁移与回滚故事就上线 v1.6** — 必须做 schema version、doctor/repair 升级、legacy reader、stale Base/template 检测与可逆重建。 +All researchers independently flagged these — they represent consensus on highest-risk items: + +1. **Mode detection ordering regression** — `deep-reading.md` carries the same `zotero_key` frontmatter as the formal note. If the filename check isn't placed FIRST in the `.md` handler (before the `zotero_key` check), the dashboard enters per-paper mode instead of deep-reading mode. **Fix:** Insert `if (activeFile.basename === 'deep-reading')` as the first branch inside the `.md` handler. + +2. **Encoding corruption on Windows CJK systems** — Python may write GBK-encoded files while Node.js reads as UTF-8, producing mojibake for Chinese Q&A content. This is a well-documented Obsidian + Windows + Chinese locale problem. **Fix:** Python must use `open(path, 'w', encoding='utf-8')` explicitly; JS reads must match. Use `PYTHONIOENCODING=utf-8` and `PYTHONUTF8=1` environment variables for child processes. + +3. **`discussion.json` schema version missing** — Shipping without `schema_version` repeats the `formal-library.json` v1→v2 migration pain from v1.6/v1.7. Every format change becomes a breaking change. **Fix:** Include `"schema_version": "1"` in the envelope from day one. Always use object envelope, never top-level array. + +4. **`btoa()` crash on Chinese filenames** — `window.btoa()` only supports Latin1 characters. If any path construction for `ai/` directory uses `btoa()` with Chinese paper titles, it throws `InvalidCharacterError`. **Fix:** Ban `btoa()` and `atob()` from the plugin. Use `Buffer.from(str, 'utf-8').toString('base64')` instead. + +5. **`active-leaf-change` double-firing mode oscillation** — Obsidian fires this event twice during tab switches (old leaf blur + new leaf focus). The 300ms debounce can catch either, causing a visible flash of global mode between per-paper and deep-reading modes. **Fix:** Extract `_resolveModeForFile()` as a pure function; guard with identity check (same mode AND same file path = no-op); increase debounce to 500ms during transitions. + +## Researcher Conflicts Resolved + +The four researchers produced high-quality, mostly aligned outputs. However, two substantive conflicts emerged: + +### Conflict 1: `discussion.json` schema shape + +| Researcher | Schema Proposal | +|------------|----------------| +| **STACK.md** | Flat `history[]` array with `index`, `timestamp`, `question`, `answer`, `tags`, `agent_model`. Summary at top level with `total_qa`, `last_updated`, `top_tags`. | +| **FEATURES.md** | Nested `sessions[]` with `session_id`, `agent`, `started`, and `qa_pairs[]` array. Each QA pair has `question`, `answer`, `source`, `timestamp`. | +| **ARCHITECTURE.md** | Similar to FEATURES: `sessions[]` with `session_id`, `timestamp`, `model`, `command`, `summary`, `message_count`, `messages[]` with `role`/`content`. Separate `format_version` (not `schema_version`). | +| **PITFALLS.md** | Warns against bare array format, recommends `schema_version` envelope. | + +**Resolution: Adopt the FEATURES.md sessions-based schema with these adjustments:** +- Use `schema_version: "1"` (from PITFALLS) — not `format_version` +- Keep session grouping (`sessions[]` → `qa_pairs[]`) — this is the right semantic model; a flat history loses the session boundary that `/pf-paper` vs `/pf-deep` invocations represent +- Use `timestamp` per QA pair (from FEATURES/STACK) — individual message timing matters for chronology +- Include `source` field as optional (from FEATURES) — traces answers back to specific Pass/section +- Add `model` at session level (from STACK/ARCHITECTURE) — the model is per-session, not per-QA +- Drop `message_count` (ARCHITECTURE proposes it but it's derivable from `qa_pairs.length`) + +**Rationale:** Sessions are the natural grouping unit — each `/pf-paper` or `/pf-deep` invocation creates a new session. A flat history array loses this structure. The FEATURES.md schema best captures this while staying minimal. + +### Conflict 2: How the plugin reads `discussion.json` + +| Researcher | Recommendation | +|------------|----------------| +| **STACK.md** | `fs.readFileSync()` — same pattern as `formal-library.json` reading | +| **ARCHITECTURE.md** | `fs.readFileSync()` — "same pattern for discussion.json... from ai/ directory" | +| **PITFALLS.md** | **DO NOT use `fs.readFileSync()`** — use `vault.adapter.read()` because discussion.json is vault-internal (paper workspace), not system-directory. `fs.readFileSync` bypasses vault cache, misses modify events, risks encoding issues. | +| **FEATURES.md** | Doesn't specify read method, only says "reads discussion.json" | + +**Resolution: PITFALLS.md is correct. Use `app.vault.adapter.read()` for discussion.json.** + +**Rationale:** `fs.readFileSync()` is the correct pattern for `formal-library.json` because it lives in `<system_dir>/PaperForge/indexes/` — outside the Obsidian vault, invisible to vault cache. But `discussion.json` lives in `Literature/<domain>/<key> - <Title>/ai/` — inside the vault, visible in Obsidian's file explorer. Using `fs.readFileSync()` on vault-internal files: +- Bypasses Obsidian's metadata cache +- Prevents `modify` events from triggering dashboard refresh when Python writes new discussions +- Risks encoding mismatch on Windows CJK systems (vault adapter normalizes encoding) + +The STACK.md and ARCHITECTURE.md researchers made a natural but incorrect assumption that the existing pattern extends unmodified. The PITFALLS researcher caught the distinction: system-directory files use `fs`, vault-internal files use `vault.adapter`. ## Implications for Roadmap -基于四份研究，v1.6 最合适的 milestone 结构不是“先做 dashboard 再补后端”，而是“先收敛真相模型，再构建 canonical index，再统一 health/status，最后再把 AI 入口挂上去”。下面的阶段划分最贴合当前 PaperForge 架构与 brownfield 风险控制。 +Based on dependency analysis across all four research files, the research converges on a **7-phase build order**. Phases 31a and 31b are quick-win bug fixes; Phase 32 establishes the mode detection infrastructure; Phase 33 builds the dashboard renderer; Phase 34 adds navigation; Phases 35-36 build the data pipeline end-to-end. -### Phase 1: 真相模型与配置收口 -**Rationale:** 这是后续所有工作最强依赖；如果配置、意图、机器事实、派生状态的边界不先锁定，后面每个功能都会放大漂移。 -**Delivers:** `paperforge.json` 的 schema_version、`vault_config` 作为规范写入目标、plugin 配置只作镜像缓存、字段 ownership matrix、术语表。 -**Addresses:** 单一配置真相、stable per-asset schema 的前置部分、idempotent rebuild 的制度基础。 -**Avoids:** 多重真相源、状态机混层、术语漂移。 +### Phase 31a: Fix Version Number Display -### Phase 2: Canonical asset index 演进与重建管线 -**Rationale:** canonical index 是 table stakes 中最核心的底座，也是 dashboard、queue、maturity、context pack 的共同依赖。 -**Delivers:** 演进版 `formal-library.json` envelope、legacy tolerant reader、`asset_index.py`、按 key 增量刷新、原子写入与文件锁。 -**Uses:** Pydantic、filelock、`tempfile` + `os.replace()`、现有 sync/ocr/deep-reading/repair 流程。 -**Implements:** 从 source artifacts 到 derived read model 的唯一投影层。 -**Avoids:** index 被当数据库、非幂等 rebuild、并发写入损坏。 +**Rationale:** Lowest risk, no dependencies. Single change in Python (`build_envelope()` adds `paperforge_version`) + single change in JS (`_fetchStats()` reads it). Quick win that improves perceived quality before new features ship. -### Phase 3: 统一 lifecycle / health / next-step 引擎 -**Rationale:** 没有统一状态与健康规则，queue、doctor、status、plugin 仍会各自解释同一篇论文。 -**Delivers:** `asset_state.py`、显式 lifecycle/readiness 模型、PDF/OCR/path/note/template/base 健康检查、evidence-aware diagnostics、镜像回写 `library-record` 的展示字段。 -**Addresses:** health surfaces、derived queue views、actionable diagnostics、maturity 的规则基础。 -**Avoids:** 浅层 health 检查、聚合状态不指向具体修复路径、CLI/plugin 语义不一致。 +**Delivers:** Version badge shows actual PaperForge version (e.g., `v1.8.0`) instead of `v—` placeholder. -### Phase 4: Status / Repair / Plugin / Bases 收敛 -**Rationale:** 只有在 canonical contract 稳定后，前端展示与修复入口才能避免锁死错误后端假设。 -**Delivers:** `status --json` 读取 canonical summary、`repair` 修源后重建 index、plugin dashboard 只消费 Python JSON、Bases 增加 `asset_state` / `library_health` / `maturity_level` / `next_step` 等列。 -**Addresses:** thin-shell dashboard、workflow progression、可操作 queue 与库级总览。 -**Avoids:** dashboard-first 反向冻结后端、plugin 重算业务规则、stale generated surfaces 被忽略。 +**Addresses:** Bug Fix: Version Number (FEATURES.md) +**Avoids:** Pitfall 8 — version badge race condition (reads from plugin manifest as floor, Python index as ceiling) -### Phase 5: Maturity guidance 与通用 AI context packs -**Rationale:** 这是真正的 differentiator，但只能建立在索引、健康与 provenance 已可靠之后；否则 AI 入口只会包装不可信资产。 -**Delivers:** ask-this-paper、ask-this-collection、copy-context-pack、manifest + context.md、level/score + next-step guidance、token budget 与 provenance 显示。 -**Addresses:** AI context entry points、traceable provenance bundle、rule-based maturity guidance。 -**Avoids:** 黑箱 AI feature、过重 context pack、学科专用 schema 侵入 core model。 +**Stack elements used:** Python `__version__` from `paperforge/__init__.py`; JS `_cachedStats.version` read path. -### Phase 6: Brownfield rollout、迁移验证与回滚保障 -**Rationale:** v1.6 面向已有 vault 与已有用户，必须把升级安全性当作交付物，而不是上线后补救项。 -**Delivers:** doctor/repair 升级、schema/version 检测、旧格式兼容、stale Base/template 检测、可逆 rebuild 流程、发布验证清单。 -**Addresses:** 长期可维护性、升级可恢复性、对既有数据的兼容性保障。 -**Avoids:** 旧库升级后 dashboard 损坏、用户需要手工删文件恢复、文档只能建议“重装试试”。 +### Phase 31b: Fix "ai" Row Bug + +**Rationale:** Independent bug fix, but must be applied AFTER Phase 31a because understanding which UI element is "ai" requires reading current dashboard render paths. Grep all render paths before removal. + +**Delivers:** Dashboard no longer shows a meaningless "ai" row. + +**Addresses:** Bug Fix: Remove meaningless "ai" row (FEATURES.md) +**Avoids:** Pitfall 9 — "ai" row removal without checking all render paths + +**Stack elements used:** `grep -rn 'ai' paperforge/plugin/main.js paperforge/plugin/styles.css` to identify source; surgical removal scoped to the specific render path. + +### Phase 32: Add Deep-Reading Mode Detection + +**Rationale:** This is the architectural foundation for everything in v1.8. Must be built before any deep-reading rendering can happen. Dependencies: nothing (pure JS change in `_detectAndSwitch()`). + +**Delivers:** When user opens `Literature/<domain>/<key> - <Title>/deep-reading.md`, the plugin detects it and dispatches to `deep-reading` mode (even though the stub renderer is a placeholder until Phase 33). + +**Implements:** Architecture component — `_detectAndSwitch()` extension, `_switchMode()` `case 'deep-reading'` branch. + +**Avoids:** Pitfall 1 (mode detection ordering — check `basename === 'deep-reading'` BEFORE `zotero_key` frontmatter); Pitfall 5 (active-leaf-change double-fire — extract `_resolveModeForFile()` as pure function with identity guard); Pitfall 3 (filename-based heuristic — verify parent directory matches `{8-char key} - {slug}` pattern, not just basename). + +**Key implementation constraint:** +```javascript +// Inside .md handler, BEFORE zotero_key check: +if (activeFile.basename === 'deep-reading') { + const parentDir = activeFile.parent?.name || ''; + const match = parentDir.match(/^([A-Z0-9]{8})\s+-\s+(.+)$/); + if (match) { + this._currentPaperKey = match[1]; + this._currentPaperEntry = this._findEntry(match[1]); + this._switchMode('deep-reading'); + return; + } +} +// THEN: existing zotero_key check for per-paper mode +``` + +### Phase 33: Build `_renderDeepReadingMode()` Component + +**Rationale:** Depends on Phase 32 (mode detection must route to this renderer). This is the largest single JS change. Must implement the sub-components (status bar, paper info header, Pass 1 summary, AI Q&A history placeholder, navigation) with empty-state handling for all data sources. + +**Delivers:** Deep-reading dashboard that shows: +- Status bar with lifecycle/OCR/deep-reading/maturity badges +- Paper info header (title, authors, year, domain) +- Pass 1 summary extracted from deep-reading.md +- AI Q&A history section (placeholder until Phase 36; shows empty state) +- Navigation link back to the per-paper dashboard + +**Implements:** Architecture component — `_renderDeepReadingMode()` with all sub-renderers. + +**Avoids:** Pitfall 6 (empty discussion.json states — implement `_loadDiscussionData()` helper returning discriminated union for all 4 empty states: `no_ai_dir`, `not_found`, `empty`, `no_discussions`, `ok`); Pitfall 13 (CSS namespace collision — use `paperforge-deepreading-*` prefix, scope under `.paperforge-mode-deepreading` wrapper class); Pitfall 3 (anti-pattern: deep-reading dashboard replacing deep-reading.md — dashboard is index/summary only, never authoritative). + +**CSS additions** (all scoped under `.paperforge-mode-deepreading`): +- `.paperforge-deep-reading-view` — root container (flex column, gap: 20px) +- `.paperforge-dr-status-bar` — Pass 1/2/3 completion indicator row +- `.paperforge-dr-pass-summary` — Pass 1 summary card +- `.paperforge-dr-discussion-card` — individual Q&A card +- `.paperforge-dr-discussion-list` — scrollable card container +- `.paperforge-dr-tag-chip` — tag pills (font-size: 10px, border-radius: 8px) +- `.paperforge-dr-empty` — empty state styling +- `.paperforge-dr-section-title` — section headers + +### Phase 34: Add "Jump to Deep Reading" Button + +**Rationale:** Depends on Phase 32 (deep-reading path resolution) and Phase 33 (deep-reading dashboard must exist for navigation to matter). Simple modification to `_renderPaperMode()`. + +**Delivers:** On per-paper dashboard card, a "Jump to Deep Reading" button appears when `entry.deep_reading_path` is non-empty and `entry.deep_reading_status === 'done'`. Click navigates to `deep-reading.md`, which triggers Phase 32 mode detection. + +**Implements:** Architecture component — integration point on per-paper dashboard contextual actions row. + +**Avoids:** Pitfall 7 (Jump button assumes file exists — verify `getAbstractFileByPath()` before `openLinkText()`, show clear `Notice` on missing file); Anti-feature (dashboard replacing deep-reading.md — button opens the actual file, dashboard is a view layer). + +**Condition to show button:** `entry.deep_reading_path && entry.deep_reading_status === 'done'` — don't show for papers that haven't had deep reading performed. + +### Phase 35: AI Discussion Recorder (Python) + +**Rationale:** Depends on nothing (standalone Python module). Can be built in parallel with Phases 32-34. Writes the data that Phase 36 will read. Must implement atomic writes to prevent corruption during concurrent access. + +**Delivers:** `paperforge/worker/discussion.py` with: +- `record_discussion(key, vault, agent, qa_pairs)` — writes both `discussion.md` and `discussion.json` +- `load_discussion_json(key, vault)` — reads existing record +- `append_discussion(key, vault, qa_pair)` — appends to existing (read-modify-write atomic via tempfile + os.replace) + +**Implements:** Architecture component — Python discussion recording module, integration with `/pf-paper` and `/pf-deep` Agent commands. + +**Uses:** Existing stdlib only (`json`, `pathlib`, `datetime`, `tempfile`, `os`); same atomic write pattern as `asset_index.py`. + +**Avoids:** Pitfall 3 (schema version missing — ship with `schema_version: "1"` in envelope); Pitfall 2 (encoding corruption — use `encoding='utf-8'` explicitly, set `PYTHONIOENCODING=utf-8` env var); Pitfall 4 (btoa on Chinese — path resolution uses zotero_key only, never writes slugified title paths); Pitfall 10 (.md vs .json inconsistency — define .json as canonical, .md as derived view); Pitfall 11 (tabs/newlines breaking Obsidian callouts — use `newline='\n'` consistently, avoid tabs); Pitfall 15 (heading collisions — use unique heading prefix `## AI Discussions` for discussion.md). + +**File operations must be atomic:** Use the proven `tempfile.NamedTemporaryFile` + `os.replace()` pattern to prevent partial writes during concurrent access. Do NOT write directly to the target file. + +### Phase 36: Wire AI Q&A History into Deep-Reading Dashboard + +**Rationale:** Depends on Phase 33 (renderer exists) and Phase 35 (data exists). This is the integration step that connects the data pipeline end-to-end. + +**Delivers:** When deep-reading dashboard shows, the AI Q&A History section renders actual discussion data from `ai/discussion.json` (last 3 Q&A pairs across all sessions, most recent first). Empty state shown when no discussions exist. + +**Implements:** Architecture component — `_renderDiscussionHistory()` in deep-reading mode renderer. + +**Avoids:** Pitfall 2 (fs.readFileSync bypasses vault API — use `app.vault.adapter.read()` for vault-internal discussion.json, NOT `fs.readFileSync`); Pitfall 12 (debounce timer leak — add 2-second cooldown after refresh, use mtime comparison to avoid redundant re-renders). + +**Key integration detail:** The modify event filter must include `discussion.json` paths to trigger dashboard refresh when Python appends new Q&A: +```javascript +const modifyHandler = this.app.vault.on('modify', (file) => { + if (file?.path?.endsWith('formal-library.json') || file?.path?.endsWith('discussion.json')) { + this._invalidateIndex(); + this._refreshCurrentMode(); + } +}); +``` + +### Phase 37: Integration Testing & Polish + +**Rationale:** After all components are built, verify end-to-end flow and fix edge cases. Must test: empty states, Chinese filenames/content, split-pane mode switching, rapid Q&A recording. + +**Delivers:** Verified end-to-end flow: `/pf-deep` → discussion files written → deep-reading dashboard shows Q&A history → Jump button navigates correctly → mode doesn't oscillate in split panes. + +**Avoids:** All pitfalls together — uses the "Looks Done But Isn't" checklist from PITFALLS.md (13 verification items). ### Phase Ordering Rationale -- **先真相模型、后界面层**：因为当前 PaperForge 已经有 worker/plugin 多表面并存，若不先统一字段归属，dashboard 只会把旧分歧可视化。 -- **先 canonical index、后 maturity/context**：因为 features 研究明确指出 queue、health、AI pack 都依赖 canonical index 和 stable schema。 -- **先 health/repair、后 AI 入口**：因为“AI-ready”必须建立在可解释、可修复、可追溯的资产状态上，而不是包装坏资产。 -- **把 plugin 放在 Phase 4 而不是更早**：这是直接响应 architecture 与 pitfalls 的共同结论，避免 UI 反向冻结错误契约。 -- **单独留 rollout phase**：因为 v1.6 是 brownfield 演进，不是 greenfield 新产品，迁移安全本身就是 milestone 范围的一部分。 +- **Fixes first (31a, 31b):** Low risk, no dependencies, quick wins that improve perceived quality before new features ship. Version fix also establishes the Python→JS version bridge that Phase 33 needs. +- **Detection before rendering (32 → 33):** Mode detection is the routing infrastructure; the renderer can't work without it. Building detection first allows incremental testing. +- **Renderer before data integration (33 → 36):** Build the UI with proper empty states first, then wire real data in. This ensures graceful degradation when no discussions exist. +- **Python recorder parallel to JS dashboard (35 parallel to 32-34):** No runtime dependency — the recorder writes files, the dashboard reads them. Can be developed in parallel. +- **Integration last (36 → 37):** Wire the full data pipeline only after both ends exist. Test end-to-end only after wiring. ### Research Flags -Phases likely needing deeper research during planning: -- **Phase 5（Maturity + Context Packs）:** 需要进一步约束 token budget、pack 结构、collection 聚合粒度，以及与 `/pf-deep`、NotebookLM 风格工作流的衔接细节。 -- **Phase 6（Brownfield rollout）:** 需要针对真实旧 vault 样本验证 legacy `formal-library.json`、旧 Base 模板、旧 plugin data 的兼容策略与回滚步骤。 +**Phases likely needing deeper research during planning:** +- **Phase 32 (Mode Detection):** The `active-leaf-change` double-fire behavior varies by Obsidian version and platform. May need platform-specific debounce tuning during implementation. Consider `/gsd-research-phase` if Obsidian API behavior is uncertain. +- **Phase 35 (AI Discussion Recorder):** The exact integration point with `/pf-paper` and `/pf-deep` scripts needs implementation-level detail. How does the Agent session signal completion? What's the handoff protocol? May need `/gsd-research-phase` if the Agent integration surface is unclear. -Phases with standard patterns（可少做专项 research-phase，直接进入需求拆解）: -- **Phase 1（真相模型与配置收口）:** 主要基于现有代码结构与已识别漂移问题，模式清晰。 -- **Phase 2（Canonical index）:** 文件投影 + schema version + 原子写入 + 文件锁属于成熟工程模式，研究结论一致。 -- **Phase 3（统一 health/status 规则）:** 规则虽需细化，但总体边界已充分明确，重点在实现而非再探索方向。 -- **Phase 4（Status/Plugin/Bases 收敛）:** 在 thin-shell 原则已明确的前提下，可按契约驱动开发，无需重新定义产品方向。 +**Phases with standard patterns (skip research-phase):** +- **Phase 31a (Version Fix):** Well-understood pattern — add field to envelope, read in fetch. Standard PaperForge index pattern. +- **Phase 33 (Dashboard Rendering):** Pattern established across 3 existing modes (global, paper, collection). Deep-reading mode follows the same `_renderXMode()` template. +- **Phase 34 (Jump Button):** Established contextual button pattern (`paperforge-contextual-btn` class) with `openLinkText()` navigation. Used elsewhere in `_renderPaperMode()`. +- **Phase 36 (Data Wiring):** Reading a JSON file and rendering DOM. Standard pattern used by `_fetchStats()` and all existing renderers. ## Confidence Assessment | Area | Confidence | Notes | |------|------------|-------| -| Stack | HIGH | 以官方文档、PyPI 版本信息与现有仓库结构为主，结论集中且依赖收敛。 | -| Features | HIGH | 直接对齐 `.planning/PROJECT.md`、Zotero/Obsidian 官方产品边界与本项目 local-first 方向。 | -| Architecture | HIGH | 基于现有 `formal-library.json` 路径契约、sync/ocr/status/plugin 真实代码边界，建议非常贴合当前仓库。 | -| Pitfalls | HIGH | 与本项目既有漂移经验、plugin thin-shell 历史教训和 brownfield 风险高度一致，且可操作性强。 | +| Stack | **HIGH** | No new dependencies. All additions are new modules/files in existing well-understood architecture. Verified against actual source code at `paperforge/plugin/main.js` (2067 lines), `paperforge/plugin/styles.css` (1325 lines), `paperforge/worker/asset_index.py` (577 lines), `paperforge/worker/sync.py` (1829 lines). Actual test fixtures confirm `ai_path` field already exists. | +| Features | **HIGH** | Feature landscape verified against Obsidian plugin ecosystem (Smart Chat, Copilot, Gemini Scribe, Claude Sessions — 5 plugins analyzed). MVP definition maps directly to v1.8 milestone requirements from PROJECT.md. Anti-features identified by consensus (3 researchers independently flagged auto-recording as a problem). | +| Architecture | **HIGH** | All integration points verified against existing source code. Mode detection hierarchy, file ownership boundaries (Python vs JS), and data flow confirmed by code inspection at specific line ranges. Canonical index schema (`formal-library.json` v2) already carries `ai_path` field. Build order derived from dependency graph analysis. | +| Pitfalls | **HIGH** | 15 pitfalls identified across 3 severity tiers. Top 5 critical pitfalls each have verified root causes from Obsidian Forum (#31841, #91927), opencode-obsidian Issue #28, nodejs/undici Issue #5002, and paperclipai Issue #3940. Recovery strategies estimated for each pitfall. "Looks Done But Isn't" checklist provides 13 concrete verification items. | -**Overall confidence:** HIGH +**Overall confidence: HIGH** + +All four researchers worked from the same source code (verified at specific line ranges), the same project context (PROJECT.md, STATE.md, AGENTS.md), and came to convergent conclusions. The two conflicts (schema shape, file read strategy) are well-characterized and resolved above. No areas of fundamental disagreement or missing information remain. ### Gaps to Address -- **`formal-library.json` 现网内容规模与刷新性能阈值**：规划时需要确认全量重建与按 key 增量刷新在真实库规模下的成本，以决定 Phase 2/3 是否同时做 query endpoint。 -- **library-record 镜像字段的最小集合**：需要在需求阶段明确哪些字段必须回写 frontmatter 供 Bases 使用，哪些只保留在 canonical index 中，避免 frontmatter 膨胀。 -- **context pack 的缓存策略**：需确认 per-paper/per-collection pack 何时惰性生成、何时缓存刷新，以避免 Phase 5 出现过重/过 eager 产物。 -- **成熟度评分展示形式**：研究更偏向 level/band，但具体是否保留 numeric score、如何解释 delta，仍需在 milestone 需求中定型。 -- **升级覆盖面验证样本**：需要至少准备几类真实旧 vault（路径异常、OCR 残缺、旧 Base 模板、旧 plugin data）作为 Phase 6 验证基线。 +- **Agent integration surface for discussion recording:** The exact API/callback for `discussion_recorder.record_session()` when an Agent session completes needs to be confirmed during Phase 35 implementation. Does `/pf-paper` emit a completion event? Where does the Q&A pair extraction happen? This is an implementation detail, not a research gap — it can be resolved during planning. +- **Deep-reading.md content parsing robustness:** The Pass 1 summary extraction relies on parsing `**一句话总览**` markers from deep-reading.md. If the agent generates different formatting in edge cases (non-standard headings, multiline bold), parsing could fail. Phase 33 should include a regex fallback that handles variations. +- **Performance at scale:** Current linear scan of index items for `_findEntry()` is O(n) and acceptable for ~100 papers. If the library grows past ~1000 papers, the dashboard may lag on mode switch. This is a known gap documented in ARCHITECTURE.md — address in a future performance phase, not v1.8. +- **Cross-platform UTF-8 path handling:** The research covers Windows CJK encoding issues thoroughly, but Linux/macOS with non-UTF-8 filesystem encodings (rare edge case) is not covered. No known users on these systems — defer until reported. ## Sources ### Primary (HIGH confidence) -- `STACK.md` — Pydantic、filelock、jsonschema、原子写入策略与 Python-first 栈建议 -- `FEATURES.md` — v1.6 table stakes、differentiators、anti-features、依赖关系 -- `ARCHITECTURE.md` — `formal-library.json` 演进方案、数据归属边界、模块拆分、阶段顺序 -- `PITFALLS.md` — brownfield 风险、迁移约束、health/maturity/plugin 反模式 -- 现有仓库代码检查：`paperforge/config.py`、`paperforge/worker/sync.py`、`paperforge/worker/ocr.py`、`paperforge/worker/status.py`、`paperforge/plugin/main.js` +- **PaperForge source code** (`paperforge/plugin/main.js` lines 1-2067, `paperforge/plugin/styles.css` lines 1-1325, `paperforge/worker/asset_index.py` lines 1-577, `paperforge/worker/sync.py` lines 1677-1749, `paperforge/worker/asset_state.py` lines 1-243) — verified mode detection, switch, rendering, index building, workspace migration, lifecycle/health/maturity computation +- **PaperForge project context** (`.planning/PROJECT.md`, `.planning/STATE.md`, `AGENTS.md`) — v1.8 milestone definition, thin-shell constraint, bug reports +- **Obsidian Developer Docs** (Context7: `/obsidianmd/obsidian-developer-docs`) — Event system, `registerEvent()`, `active-leaf-change`, `debounce()` best practices +- **Obsidian Forum #31841** — `active-leaf-change` double-fire behavior confirmed by multiple plugin developers +- **Obsidian Forum #91927** — GB2312 to UTF-8 conversion corrupting Chinese files — encoding mismatch pattern +- **opencode-obsidian Issue #28** — `btoa()` crash with Chinese characters — verified fix: `Buffer.from(str).toString('base64')` ### Secondary (MEDIUM confidence) -- Zotero 官方文档（collections/tags、attachments、duplicates）— 用于确认上游真相源边界与“不替代 Zotero”的产品定位 -- NotebookLM、ResearchRabbit、Litmaps、SciSpace 的产品定位信息 — 用于界定 differentiator 与 anti-feature 边界 +- **Obsidian Smart Chat** — Chat thread linking, Dataview dashboards, chat-active/chat-done tracking +- **Obsidian Copilot (DeepWiki)** — Chat persistence and history system with markdown files, YAML frontmatter, session grouping +- **Claude Sessions plugin** — Session timeline rendering, summary dashboard, Obsidian Bases dashboards +- **Gemini Scribe** — Per-note history file pattern, auto-appending +- **nodejs/undici Issue #5002** — Multi-byte UTF-8 character corruption at chunk boundaries with CJK text +- **paperclipai/paperclip Issue #3940** — GBK-UTF8 encoding mismatch on Windows with CJK child process output +- **Excalidraw plugin commit 5c628e0** — Real-world debounce timer cleanup in Obsidian plugin `onunload` +- **obsidian-current-view commit ad110f7** — Replacing requestAnimationFrame with setTimeout debounce ### Tertiary (LOW confidence) -- AI knowledge/RAG 一般性文章 — 仅作为 context pack explainability 与 retrieval governance 的启发，不作为核心架构依据 +- **PaulGP llms.txt proposal** — AI-readable paper annotations design philosophy (context only, not implementation reference) +- **Effortless Academic discussion writing guide** — Q&A-style paper discussion structure (practitioner guide, not technical reference) +- **Templater Issue #1629** — `app.vault.modify()` race condition with multiple handlers (relevant but unverified for PaperForge's specific use case) --- -*Research completed: 2026-05-03* + +*Research completed: 2026-05-06* *Ready for roadmap: yes* +*Conflicts resolved: 2 (schema shape → sessions-based; file read strategy → vault.adapter.read for vault-internal files)* +*Gates cleared: Zero new dependencies, zero CLI command changes, thin-shell principle preserved* diff --git a/paperforge/plugin/main.js b/paperforge/plugin/main.js index 32f82eea..5a8326bf 100644 --- a/paperforge/plugin/main.js +++ b/paperforge/plugin/main.js @@ -15,8 +15,8 @@ const PF_RIBBON_SVG = ` // ── i18n: language pack (auto-detected from Obsidian config) ── const LANG = { - en: { header_title:'PaperForge',desc:'Obsidian + Zotero literature pipeline.',setup_done:'✓ PaperForge environment configured',setup_pending:'Not installed — complete preparation and open the wizard',section_prep:'Prerequisites',section_prep_desc:'Before first use, complete the following:',section_guide:'Usage',section_config:'Configuration',prep_python:'Python 3.9+',prep_python_desc:'Must be callable from command line. Click below to auto-detect.',prep_zotero:'Zotero Desktop',prep_zotero_desc:'Install Zotero (https://www.zotero.org)',prep_bbt:'Better BibTeX',prep_bbt_desc:'Zotero → Tools → Add-ons → Install Better BibTeX',prep_export:'BBT Auto-export',prep_export_desc:'Right-click collection → Export → BetterBibTeX JSON → Keep updated → to:',prep_key:'PaddleOCR Key',prep_key_desc:'Get free key at https://aistudio.baidu.com/paddleocr',guide_open:'Open Dashboard',guide_open_desc:'Ctrl+P → "PaperForge: Open Dashboard", or sidebar book icon',guide_sync:'Sync Literature',guide_sync_desc:'Dashboard → Sync Library — pull from Zotero, generate notes',guide_ocr:'Run OCR',guide_ocr_desc:'Dashboard → Run OCR — extract PDF text & figures',btn_install:'Open Wizard',btn_reconfig:'Reconfigure',btn_install_desc:'Auto-detect environment, then open setup wizard',btn_reconfig_desc:'Re-run wizard to change directories or keys',wizard_step1:'Overview',wizard_step2:'Dirs',wizard_step3:'Agent',wizard_step4:'Install',wizard_step5:'Done',wizard_title:'PaperForge Setup Wizard',wizard_intro:'This wizard will guide you through the complete setup.',wizard_dir_hint:'The resources directory is the root for all literature data. Sub-directories inside it:',wizard_dir_sub_hint:'Two sub-directories within resources:',wizard_sys_hint:'System directories (at vault root):',wizard_agent_hint:'Select your AI Agent platform. Skill files deploy in the correct format.',wizard_keys_hint:'API key and Zotero:',wizard_preview:'System/agent files at vault root. Literature (notes, index) under resources.',dir_vault:'Vault Path',dir_resources:'Resource Dir',dir_notes:'Notes Dir',dir_index:'Index Dir',dir_system:'System Dir',dir_base:'Base Dir',field_paddleocr:'PaddleOCR API Key',field_zotero_data:'Zotero Data Dir',field_zotero_placeholder:'Optional, for auto PDF detection',label_agent:'Agent Platform',check_python_ok:'Ready',check_python_fail:'Not found',check_zotero_ok:'Found',check_zotero_fail:'Not detected',check_bbt_ok:'Installed',check_bbt_fail:'Not detected',install_btn:'Install',install_btn_running:'Installing...',install_btn_retry:'Retry',install_complete:'✓ Installation complete!',install_failed:'✗ Installation failed: ',complete_title:'✓ Setup Complete',complete_summary:'Configuration',complete_next:'Next Steps',complete_step1:'Open Dashboard',complete_step1_desc:'Ctrl+P → "PaperForge: Open Dashboard" or sidebar book icon',complete_step2:'Sync Literature',complete_step2_desc:'Dashboard → Sync Library — pull from Zotero',complete_step3:'Run OCR',complete_step3_desc:'Dashboard → Run OCR — extract full text & figures',complete_step4:'Configure BBT Auto-export',nav_prev:'← Back',nav_next:'Next →',nav_close:'Close',validate_fail:'Validation failed',validate_vault:'Vault path not set',validate_resources:'Resource dir not set',validate_notes:'Notes dir not set',validate_index:'Index dir not set',validate_base:'Base dir not set',validate_key:'API key not set',validate_system:'System dir not set',notice_python_missing:'Python not detected. Install Python 3.9+ and add to PATH.',notice_check_fail:'Missing: ',panel_actions:'Quick Actions',action_running:'Running ',api_key_set:'Configured ✓',api_key_missing:'Not configured ✗',not_set:'Not set', }, - zh: { header_title:'PaperForge',desc:'Obsidian + Zotero 文献管理流水线。自动同步文献、生成笔记、OCR 提取全文，一站式文献精读工作流。',setup_done:'✓ PaperForge 环境已配置完成',setup_pending:'尚未安装，完成安装准备后点击安装向导',section_prep:'安装准备',section_prep_desc:'首次使用前，请依次完成以下准备：',section_guide:'操作方式',section_config:'当前配置',prep_python:'Python 3.9+',prep_python_desc:'确保 Python 可命令行调用。点击下方按钮自动检测。',prep_zotero:'Zotero 桌面版',prep_zotero_desc:'安装 Zotero (https://www.zotero.org)',prep_bbt:'Better BibTeX',prep_bbt_desc:'Zotero → 工具 → 插件 → 安装 Better BibTeX',prep_export:'BBT 自动导出',prep_export_desc:'右键文献子分类 → 导出分类 → BetterBibTeX JSON → 勾选保持更新 → 导出到（JSON 文件名即为 Base 名）：',prep_key:'PaddleOCR Key',prep_key_desc:'在 https://aistudio.baidu.com/paddleocr 获取 API Key',guide_open:'打开 Dashboard',guide_open_desc:'Ctrl+P → 输入 PaperForge: Open Dashboard，或点左侧书本图标',guide_sync:'同步文献',guide_sync_desc:'Dashboard 中点 Sync Library，从 Zotero 拉取文献生成笔记',guide_ocr:'运行 OCR',guide_ocr_desc:'Dashboard 中点 Run OCR，提取 PDF 全文与图表',btn_install:'打开安装向导',btn_reconfig:'重新配置',btn_install_desc:'自动检测 Python + 前置环境，通过后打开分步安装向导',btn_reconfig_desc:'重新运行安装向导，修改目录或密钥配置',wizard_step1:'概览',wizard_step2:'目录',wizard_step3:'Agent',wizard_step4:'安装',wizard_step5:'完成',wizard_title:'PaperForge 安装向导',wizard_intro:'本向导将引导您完成 PaperForge 环境的完整配置。安装过程会自动创建所有目录结构，无需手动操作。',wizard_dir_hint:'资源目录是文献数据的统一根目录，以下子目录将创建在其内部：',wizard_dir_sub_hint:'资源目录内的两个子目录：',wizard_sys_hint:'独立于资源目录的系统文件：',wizard_agent_hint:'选择你使用的 AI Agent 平台，安装时将按对应格式部署技能文件：',wizard_keys_hint:'以下为 API 密钥与 Zotero 配置：',wizard_preview:'系统文件和 Agent 配置位于 Vault 根目录下。文献数据（正文、索引）统一存放在资源目录内。安装后仍可在设置中修改。',dir_vault:'Vault 路径',dir_resources:'资源目录',dir_notes:'正文目录',dir_index:'索引目录',dir_system:'系统目录',dir_base:'Base 目录',field_paddleocr:'PaddleOCR API 密钥',field_zotero_data:'Zotero 数据目录',field_zotero_placeholder:'可选，用于自动检测 PDF',label_agent:'Agent 平台',check_python_ok:'已就绪',check_python_fail:'未安装',check_zotero_ok:'已安装',check_zotero_fail:'未检测到',check_bbt_ok:'已安装',check_bbt_fail:'未检测到',install_btn:'开始安装',install_btn_running:'正在安装...',install_btn_retry:'重试',install_complete:'✓ 安装完成！',install_failed:'✗ 安装失败：',complete_title:'✓ PaperForge 安装完成',complete_summary:'当前完整配置',complete_next:'下一步操作',complete_step1:'打开 PaperForge Dashboard',complete_step1_desc:'Ctrl+P → 输入 PaperForge: Open Dashboard，或点左侧书本图标',complete_step2:'同步文献',complete_step2_desc:'Dashboard 中点 Sync Library，从 Zotero 拉取文献生成笔记',complete_step3:'运行 OCR',complete_step3_desc:'Dashboard 中点 Run OCR，提取 PDF 全文与图表',complete_step4:'配置 BBT 自动导出',nav_prev:'← 上一步',nav_next:'下一步 →',nav_close:'关闭',validate_fail:'配置验证失败',validate_vault:'Vault 路径未填写',validate_resources:'资源目录未填写',validate_notes:'正文目录未填写',validate_index:'索引目录未填写',validate_base:'Base 目录未填写',validate_key:'PaddleOCR API 密钥未填写',validate_system:'系统目录未填写',notice_python_missing:'Python 未检测到，请先安装 Python 3.9+ 并加入 PATH',notice_check_fail:'未通过: ',panel_actions:'快捷操作',action_running:'正在执行 ',api_key_set:'已配置 ✓',api_key_missing:'未配置 ✗',not_set:'未设置', } + en: { header_title:'PaperForge',desc:'Obsidian + Zotero literature pipeline.',setup_done:'✓ PaperForge environment configured',setup_pending:'Not installed — complete preparation and open the wizard',section_prep:'Prerequisites',section_prep_desc:'Before first use, complete the following:',section_guide:'Usage',section_config:'Configuration',prep_python:'Python 3.9+',prep_python_desc:'Must be callable from command line. Click below to auto-detect.',prep_zotero:'Zotero Desktop',prep_zotero_desc:'Install Zotero (https://www.zotero.org)',prep_bbt:'Better BibTeX',prep_bbt_desc:'Zotero → Tools → Add-ons → Install Better BibTeX',prep_export:'BBT Auto-export',prep_export_desc:'Right-click collection → Export → BetterBibTeX JSON → Keep updated → to:',prep_key:'PaddleOCR Key',prep_key_desc:'Get free key at https://aistudio.baidu.com/paddleocr',guide_open:'Open Dashboard',guide_open_desc:'Ctrl+P → "PaperForge: Open Dashboard", or sidebar book icon',guide_sync:'Sync Literature',guide_sync_desc:'Dashboard → Sync Library — pull from Zotero, generate notes',guide_ocr:'Run OCR',guide_ocr_desc:'Dashboard → Run OCR — extract PDF text & figures',btn_install:'Open Wizard',btn_reconfig:'Reconfigure',btn_install_desc:'Auto-detect environment, then open setup wizard',btn_reconfig_desc:'Re-run wizard to change directories or keys',wizard_step1:'Overview',wizard_step2:'Dirs',wizard_step3:'Agent',wizard_step4:'Install',wizard_step5:'Done',wizard_title:'PaperForge Setup Wizard',wizard_intro:'This wizard will guide you through the complete setup.',wizard_dir_hint:'The resources directory is the root for all literature data. Sub-directories inside it:',wizard_dir_sub_hint:'Two sub-directories within resources:',wizard_sys_hint:'System directories (at vault root):',wizard_agent_hint:'Select your AI Agent platform. Skill files deploy in the correct format.',wizard_keys_hint:'API key and Zotero:',wizard_preview:'System/agent files at vault root. Literature (notes, index) under resources.',dir_vault:'Vault Path',dir_resources:'Resource Dir',dir_notes:'Notes Dir',dir_index:'Index Dir',dir_system:'System Dir',dir_base:'Base Dir',field_paddleocr:'PaddleOCR API Key',field_zotero_data:'Zotero Data Dir',field_zotero_placeholder:'Optional, for auto PDF detection',label_agent:'Agent Platform',check_python_ok:'Ready',check_python_fail:'Not found',check_zotero_ok:'Found',check_zotero_fail:'Not detected',check_bbt_ok:'Installed',check_bbt_fail:'Not detected',install_btn:'Install',install_btn_running:'Installing...',install_btn_retry:'Retry',install_complete:'✓ Installation complete!',install_failed:'✗ Installation failed: ',complete_title:'✓ Setup Complete',complete_summary:'Configuration',complete_next:'Next Steps',complete_step1:'Open Dashboard',complete_step1_desc:'Ctrl+P → "PaperForge: Open Dashboard" or sidebar book icon',complete_step2:'Sync Literature',complete_step2_desc:'Dashboard → Sync Library — pull from Zotero',complete_step3:'Run OCR',complete_step3_desc:'Dashboard → Run OCR — extract full text & figures',complete_step4:'Configure BBT Auto-export',nav_prev:'← Back',nav_next:'Next →',nav_close:'Close',validate_fail:'Validation failed',validate_vault:'Vault path not set',validate_resources:'Resource dir not set',validate_notes:'Notes dir not set',validate_index:'Index dir not set',validate_base:'Base dir not set',validate_key:'API key not set',validate_system:'System dir not set',notice_python_missing:'Python not detected. Install Python 3.9+ and add to PATH.',notice_check_fail:'Missing: ',panel_actions:'Quick Actions',action_running:'Running ',api_key_set:'Configured ✓',api_key_missing:'Not configured ✗',not_set:'Not set',jump_to_deep_reading:'Open Deep Reading',deep_reading_not_found:'Deep reading file not found', }, + zh: { header_title:'PaperForge',desc:'Obsidian + Zotero 文献管理流水线。自动同步文献、生成笔记、OCR 提取全文，一站式文献精读工作流。',setup_done:'✓ PaperForge 环境已配置完成',setup_pending:'尚未安装，完成安装准备后点击安装向导',section_prep:'安装准备',section_prep_desc:'首次使用前，请依次完成以下准备：',section_guide:'操作方式',section_config:'当前配置',prep_python:'Python 3.9+',prep_python_desc:'确保 Python 可命令行调用。点击下方按钮自动检测。',prep_zotero:'Zotero 桌面版',prep_zotero_desc:'安装 Zotero (https://www.zotero.org)',prep_bbt:'Better BibTeX',prep_bbt_desc:'Zotero → 工具 → 插件 → 安装 Better BibTeX',prep_export:'BBT 自动导出',prep_export_desc:'右键文献子分类 → 导出分类 → BetterBibTeX JSON → 勾选保持更新 → 导出到（JSON 文件名即为 Base 名）：',prep_key:'PaddleOCR Key',prep_key_desc:'在 https://aistudio.baidu.com/paddleocr 获取 API Key',guide_open:'打开 Dashboard',guide_open_desc:'Ctrl+P → 输入 PaperForge: Open Dashboard，或点左侧书本图标',guide_sync:'同步文献',guide_sync_desc:'Dashboard 中点 Sync Library，从 Zotero 拉取文献生成笔记',guide_ocr:'运行 OCR',guide_ocr_desc:'Dashboard 中点 Run OCR，提取 PDF 全文与图表',btn_install:'打开安装向导',btn_reconfig:'重新配置',btn_install_desc:'自动检测 Python + 前置环境，通过后打开分步安装向导',btn_reconfig_desc:'重新运行安装向导，修改目录或密钥配置',wizard_step1:'概览',wizard_step2:'目录',wizard_step3:'Agent',wizard_step4:'安装',wizard_step5:'完成',wizard_title:'PaperForge 安装向导',wizard_intro:'本向导将引导您完成 PaperForge 环境的完整配置。安装过程会自动创建所有目录结构，无需手动操作。',wizard_dir_hint:'资源目录是文献数据的统一根目录，以下子目录将创建在其内部：',wizard_dir_sub_hint:'资源目录内的两个子目录：',wizard_sys_hint:'独立于资源目录的系统文件：',wizard_agent_hint:'选择你使用的 AI Agent 平台，安装时将按对应格式部署技能文件：',wizard_keys_hint:'以下为 API 密钥与 Zotero 配置：',wizard_preview:'系统文件和 Agent 配置位于 Vault 根目录下。文献数据（正文、索引）统一存放在资源目录内。安装后仍可在设置中修改。',dir_vault:'Vault 路径',dir_resources:'资源目录',dir_notes:'正文目录',dir_index:'索引目录',dir_system:'系统目录',dir_base:'Base 目录',field_paddleocr:'PaddleOCR API 密钥',field_zotero_data:'Zotero 数据目录',field_zotero_placeholder:'可选，用于自动检测 PDF',label_agent:'Agent 平台',check_python_ok:'已就绪',check_python_fail:'未安装',check_zotero_ok:'已安装',check_zotero_fail:'未检测到',check_bbt_ok:'已安装',check_bbt_fail:'未检测到',install_btn:'开始安装',install_btn_running:'正在安装...',install_btn_retry:'重试',install_complete:'✓ 安装完成！',install_failed:'✗ 安装失败：',complete_title:'✓ PaperForge 安装完成',complete_summary:'当前完整配置',complete_next:'下一步操作',complete_step1:'打开 PaperForge Dashboard',complete_step1_desc:'Ctrl+P → 输入 PaperForge: Open Dashboard，或点左侧书本图标',complete_step2:'同步文献',complete_step2_desc:'Dashboard 中点 Sync Library，从 Zotero 拉取文献生成笔记',complete_step3:'运行 OCR',complete_step3_desc:'Dashboard 中点 Run OCR，提取 PDF 全文与图表',complete_step4:'配置 BBT 自动导出',nav_prev:'← 上一步',nav_next:'下一步 →',nav_close:'关闭',validate_fail:'配置验证失败',validate_vault:'Vault 路径未填写',validate_resources:'资源目录未填写',validate_notes:'正文目录未填写',validate_index:'索引目录未填写',validate_base:'Base 目录未填写',validate_key:'PaddleOCR API 密钥未填写',validate_system:'系统目录未填写',notice_python_missing:'Python 未检测到，请先安装 Python 3.9+ 并加入 PATH',notice_check_fail:'未通过: ',panel_actions:'快捷操作',action_running:'正在执行 ',api_key_set:'已配置 ✓',api_key_missing:'未配置 ✗',not_set:'未设置',jump_to_deep_reading:'跳转到精读',deep_reading_not_found:'精读文件未找到', } }; let T = LANG.zh; @@ -210,10 +210,11 @@ const ACTIONS = [ class PaperForgeStatusView extends ItemView { constructor(leaf) { super(leaf); - this._currentMode = null; // 'global' | 'paper' | 'collection' (D-05) + this._currentMode = null; // 'global' | 'paper' | 'collection' | 'deep-reading' (D-05) this._currentDomain = null; // domain name when in collection mode (D-15) this._currentPaperKey = null; // zotero_key when in per-paper mode (D-03) this._currentPaperEntry = null; // full entry when in per-paper mode + this._currentFilePath = null; // active file path for identity guard (D-06) this._cachedItems = null; // lazy-loaded index items (Plan 28-01) this._modeSubscribers = []; // event handler refs for cleanup this._leafChangeTimer = null; // debounce timer for active-leaf-change @@ -561,12 +562,10 @@ class PaperForgeStatusView extends ItemView { } const stages = [ - { key: 'imported', label: 'Imported' }, { key: 'indexed', label: 'Indexed' }, { key: 'pdf_ready', label: 'PDF Ready' }, { key: 'fulltext_ready', label: 'Fulltext Ready' }, - { key: 'deep_read', label: 'Deep Read' }, - { key: 'ai_ready', label: 'AI Ready' }, + { key: 'deep_read_done', label: 'Deep Read' }, ]; const stepper = container.createEl('div', { cls: 'paperforge-lifecycle-stepper' }); @@ -670,12 +669,10 @@ class PaperForgeStatusView extends ItemView { } const stages = [ - { key: 'imported', label: 'Imported', cls: 'stage-imported' }, { key: 'indexed', label: 'Indexed', cls: 'stage-indexed' }, { key: 'pdf_ready', label: 'PDF Ready', cls: 'stage-pdf-ready' }, { key: 'fulltext_ready', label: 'Fulltext Ready', cls: 'stage-fulltext-ready' }, - { key: 'deep_read', label: 'Deep Read', cls: 'stage-deep-read' }, - { key: 'ai_ready', label: 'AI Ready', cls: 'stage-ai-ready' }, + { key: 'deep_read_done', label: 'Deep Read', cls: 'stage-deep-read' }, ]; const chart = container.createEl('div', { cls: 'paperforge-bar-chart' }); @@ -714,64 +711,65 @@ class PaperForgeStatusView extends ItemView { this._cachedItems = null; } - /* ── Context Detection & Mode Switch (D-01, D-02, D-03, D-04, D-10) ── */ - _detectAndSwitch() { - const activeFile = this.app.workspace.getActiveFile(); + /* ── Pure Mode Resolution (D-07, Phase 32) ── */ + _resolveModeForFile(file) { + if (!file) return { mode: 'global', filePath: null, key: null, domain: null }; - if (!activeFile) { - // No active file -> global mode (D-04) - this._switchMode('global'); - return; - } - - const ext = activeFile.extension; + const ext = file.extension; + const filePath = file.path; if (ext === 'base') { - // .base file -> collection mode (D-02, D-15) - this._currentDomain = activeFile.basename; - this._currentPaperKey = null; - this._currentPaperEntry = null; - this._switchMode('collection'); - return; + return { mode: 'collection', filePath, key: null, domain: file.basename }; } if (ext === 'md') { - // .md file -- check for zotero_key in frontmatter (D-03) - const cache = this.app.metadataCache.getFileCache(activeFile); - const key = cache && cache.frontmatter && cache.frontmatter.zotero_key; - - if (key) { - // Has zotero_key -> per-paper mode (D-03) - this._currentPaperKey = key; - this._currentPaperEntry = this._findEntry(key); - this._currentDomain = null; - this._switchMode('paper'); - } else { - // .md without zotero_key -> global mode (D-04) - this._currentDomain = null; - this._currentPaperKey = null; - this._currentPaperEntry = null; - this._switchMode('global'); + // D-01: Check deep-reading.md FIRST — before zotero_key frontmatter + if (file.name === 'deep-reading.md') { + const parentDir = file.parent ? file.parent.name : ''; + // D-02: Parent directory must match {8-char-key} - {Title} + if (/^[A-Z0-9]{8} - .+$/.test(parentDir)) { + const cache = this.app.metadataCache.getFileCache(file); + const key = cache && cache.frontmatter && cache.frontmatter.zotero_key; + if (key) { + return { mode: 'deep-reading', filePath, key, domain: null }; + } + } + // D-03: Fall through to normal .md handling } - return; + + // Standard .md — check for zotero_key in frontmatter (D-03) + const cache = this.app.metadataCache.getFileCache(file); + const key = cache && cache.frontmatter && cache.frontmatter.zotero_key; + if (key) { + return { mode: 'paper', filePath, key, domain: null }; + } + return { mode: 'global', filePath, key: null, domain: null }; } - // Any other file type -> global mode (D-04) - this._currentDomain = null; - this._currentPaperKey = null; - this._currentPaperEntry = null; - this._switchMode('global'); + return { mode: 'global', filePath, key: null, domain: null }; + } + + /* ── Context Detection & Mode Switch (D-01, D-02, D-03, D-04, D-10) ── */ + _detectAndSwitch() { + const resolved = this._resolveModeForFile(this.app.workspace.getActiveFile()); + + this._currentDomain = resolved.domain || null; + this._currentPaperKey = resolved.key || null; + this._currentPaperEntry = resolved.key ? this._findEntry(resolved.key) : null; + + this._switchMode(resolved.mode, resolved.filePath); } /* ── Mode Switching (D-05, D-06) ── */ - _switchMode(mode) { - if (this._currentMode === mode) { - // Already in this mode -- just refresh if needed (D-04) + _switchMode(mode, filePath) { + // D-06: Identity guard — check BOTH mode AND file path + if (this._currentMode === mode && this._currentFilePath === filePath) { this._refreshCurrentMode(); return; } this._currentMode = mode; + this._currentFilePath = filePath; // Clear existing content (D-06) this._contentEl.empty(); @@ -791,6 +789,9 @@ class PaperForgeStatusView extends ItemView { case 'collection': this._renderCollectionMode(); break; + case 'deep-reading': + this._renderDeepReadingMode(); + break; } } @@ -929,13 +930,29 @@ class PaperForgeStatusView extends ItemView { }); }); } else if (nextStep === 'ready') { - // Show "Copy Context" shortcut for ready state (D-09) - const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); - trigger.createEl('span', { text: '\u2139 Copy Context' }); - trigger.addEventListener('click', () => { - const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); - if (action) this._runAction(action, trigger); - }); + if (entry.deep_reading_path && entry.deep_reading_status === 'done') { + // D-01, D-03: Jump-to-deep-reading button replaces Copy Context when deep reading exists + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\uD83D\uDD0D ' + t('jump_to_deep_reading') }); + trigger.addEventListener('click', () => { + // D-04: Follow _openFulltext() pattern — verify file, open, error Notice + const drFile = this.app.vault.getAbstractFileByPath(entry.deep_reading_path); + if (drFile) { + this.app.workspace.openLinkText(drFile.path, ''); + } else { + // D-05: File missing from disk despite index claiming it exists + new Notice('[!!] ' + t('deep_reading_not_found'), 6000); + } + }); + } else { + // D-02, D-03: Fall back to existing Copy Context button + const trigger = card.createEl('button', { cls: 'paperforge-next-step-trigger' }); + trigger.createEl('span', { text: '\u2139 Copy Context' }); + trigger.addEventListener('click', () => { + const action = ACTIONS.find(a => a.id === 'paperforge-copy-context'); + if (action) this._runAction(action, trigger); + }); + } } } @@ -955,6 +972,165 @@ class PaperForgeStatusView extends ItemView { } } + /* ── Deep-Reading Mode Render (Phase 33) ── */ + async _renderDeepReadingMode() { + const entry = this._currentPaperEntry; + const modeGuard = () => this._currentMode === 'deep-reading'; + const view = this._contentEl.createEl('div', { cls: 'paperforge-mode-deepreading' }); + + // Read deep-reading.md content + let deepReadingText = ''; + if (entry && entry.deep_reading_path) { + const drFile = this.app.vault.getAbstractFileByPath(entry.deep_reading_path); + if (drFile) { + deepReadingText = await this.app.vault.read(drFile); + } + if (!modeGuard()) return; // Guard: mode switched during async read + } + + // Read discussion.json + let discussionData = null; + if (entry && entry.ai_path) { + const aiPath = entry.ai_path.replace(/^\[\[/, '').replace(/\]\]$/, ''); + const djPath = aiPath + 'discussion.json'; + const djFile = this.app.vault.getAbstractFileByPath(djPath); + if (djFile) { + try { + const raw = await this.app.vault.read(djFile); + if (!modeGuard()) return; // Guard: mode switched + discussionData = JSON.parse(raw); + } catch (e) { /* file missing or parse error */ } + } + } + + // Render sections + this._renderDeepStatusCard(view, entry); + this._renderDeepPass1Card(view, deepReadingText); + this._renderDeepQACard(view, discussionData); + } + + /* ── Deep-Reading Status Card ── */ + _renderDeepStatusCard(container, entry) { + const card = container.createEl('div', { cls: 'paperforge-deepreading-card' }); + card.createEl('div', { cls: 'paperforge-deepreading-card-title', text: '\uD83D\uDCCA \u72B6\u6001\u6982\u89C8' }); + + if (!entry) { + card.createEl('div', { cls: 'paperforge-deepreading-empty', text: '\u6682\u65E0\u6570\u636E' }); + return; + } + + const statusItems = [ + { label: 'Figure-Map', value: entry.figure_map ? '\u2705 \u5DF2\u751F\u6210' : '\u23F3 \u5F85\u751F\u6210' }, + { label: 'OCR \u72B6\u6001', value: entry.ocr_status === 'done' ? '\u2705 \u5DF2\u5B8C\u6210' : '\u23F3 ' + (entry.ocr_status || '\u5F85\u5904\u7406') }, + { label: 'Pass \u5B8C\u6210', value: this._getPassCompletion(entry) }, + { label: '\u5065\u5EB7\u72B6\u51B5', value: entry.health && entry.health.pdf_health === 'healthy' ? '\u2705 \u6B63\u5E38' : '\u26A0\uFE0F \u9700\u5173\u6CE8' }, + ]; + + const list = card.createEl('div', { cls: 'paperforge-deepreading-status-list' }); + for (const item of statusItems) { + const row = list.createEl('div', { cls: 'paperforge-deepreading-status-row' }); + row.createEl('span', { cls: 'paperforge-deepreading-status-label', text: item.label }); + row.createEl('span', { cls: 'paperforge-deepreading-status-value', text: item.value }); + } + } + + _getPassCompletion(entry) { + const status = entry.deep_reading_status || 'pending'; + if (status === 'done') return '\u2705 3/3 (\u5DF2\u5B8C\u6210)'; + return '\u23F3 \u5F85\u5B8C\u6210'; + } + + /* ── Deep-Reading Pass 1 Summary Card ── */ + _renderDeepPass1Card(container, text) { + const card = container.createEl('div', { cls: 'paperforge-deepreading-card' }); + card.createEl('div', { cls: 'paperforge-deepreading-card-title', text: '\uD83D\uDCDD Pass 1 \u603B\u7ED3' }); + + if (!text || text.trim() === '') { + card.createEl('div', { cls: 'paperforge-deepreading-empty', text: '\u6682\u65E0 Pass 1 \u603B\u7ED3' }); + return; + } + + const extracted = this._extractPass1Content(text); + if (!extracted) { + card.createEl('div', { cls: 'paperforge-deepreading-empty', text: '\u6682\u65E0 Pass 1 \u603B\u7ED3' }); + return; + } + + const content = card.createEl('div', { cls: 'paperforge-deepreading-pass1-content' }); + const lines = extracted.split('\n').filter(l => l.trim()); + for (const line of lines) { + if (line.startsWith('### ')) { + content.createEl('h4', { cls: 'paperforge-deepreading-pass1-subheading', text: line.replace('### ', '') }); + } else if (line.startsWith('**') && line.endsWith('**')) { + content.createEl('p', { cls: 'paperforge-deepreading-pass1-marker', text: line }); + } else if (line.trim()) { + content.createEl('p', { cls: 'paperforge-deepreading-pass1-text', text: line }); + } + } + } + + _extractPass1Content(text) { + const markers = ['**\u4E00\u53E5\u8BDD\u603B\u89C8**', '## Pass 1', '**\u6587\u7AE0\u6458\u8981**']; + for (const marker of markers) { + const idx = text.indexOf(marker); + if (idx !== -1) { + const after = idx + marker.length; + // Cut at next major section marker + const cutMarkers = ['**\u8BC1\u636E\u8FB9\u754C**', '**Figure \u5BFC\u8BFB**', '**\u4E3B\u8981\u53D1\u73B0**']; + let nextCut = text.length; + for (const cm of cutMarkers) { + const ci = text.indexOf(cm, after); + if (ci !== -1 && ci < nextCut) nextCut = ci; + } + return text.substring(after, nextCut).trim(); + } + } + return null; + } + + /* ── Deep-Reading AI Q&A History Card ── */ + _renderDeepQACard(container, data) { + const card = container.createEl('div', { cls: 'paperforge-deepreading-card' }); + + // D-10: Collapsible header + const header = card.createEl('div', { cls: 'paperforge-deepreading-card-header collapsible' }); + header.createEl('div', { cls: 'paperforge-deepreading-card-title', text: '\uD83D\uDCAC AI \u95EE\u7B54\u8BB0\u5F55' }); + + const body = card.createEl('div', { cls: 'paperforge-deepreading-card-body collapsed' }); + + // Toggle collapse + header.addEventListener('click', () => { + body.classList.toggle('collapsed'); + }); + + // D-12: Empty states + if (!data || !data.sessions || data.sessions.length === 0) { + body.createEl('div', { cls: 'paperforge-deepreading-empty', text: !data ? '\u6682\u65E0\u8BA8\u8BBA\u8BB0\u5F55' : '\u6682\u65E0\u95EE\u7B54\u5185\u5BB9' }); + return; + } + + // D-08: Sessions-based grouping + for (const session of data.sessions) { + const sessionEl = body.createEl('div', { cls: 'paperforge-deepreading-session' }); + const sessionHeader = sessionEl.createEl('div', { cls: 'paperforge-deepreading-session-header' }); + sessionHeader.createEl('span', { text: session.model || 'AI' }); + sessionHeader.createEl('span', { cls: 'paperforge-deepreading-session-date', text: session.started || '' }); + + // D-09: Dialog bubbles + if (session.qa_pairs) { + for (const qa of session.qa_pairs) { + const qBubble = sessionEl.createEl('div', { cls: 'paperforge-deepreading-bubble question' }); + qBubble.createEl('div', { cls: 'bubble-label', text: '\u95EE\u9898' }); + qBubble.createEl('div', { cls: 'bubble-text', text: qa.question }); + + const aBubble = sessionEl.createEl('div', { cls: 'paperforge-deepreading-bubble answer' }); + aBubble.createEl('div', { cls: 'bubble-label', text: '\u89E3\u7B54' }); + aBubble.createEl('div', { cls: 'bubble-text', text: qa.answer }); + } + } + } + } + /* ── Collection Mode Render (Phase 30) ── */ _renderCollectionMode() { const domain = this._currentDomain || 'Unknown'; @@ -983,8 +1159,8 @@ class PaperForgeStatusView extends ItemView { const lifecycle = item.lifecycle || 'pdf_ready'; lifecycleCounts[lifecycle] = (lifecycleCounts[lifecycle] || 0) + 1; - if (['fulltext_ready', 'deep_read', 'ai_ready'].includes(lifecycle)) fulltextReady++; - if (['deep_read', 'ai_ready'].includes(lifecycle)) deepRead++; + if (['fulltext_ready', 'deep_read_done', 'ai_context_ready'].includes(lifecycle)) fulltextReady++; + if (['deep_read_done', 'ai_context_ready'].includes(lifecycle)) deepRead++; const health = item.health || {}; for (const dim of ['pdf_health', 'ocr_health', 'note_health', 'asset_health']) { @@ -1064,6 +1240,9 @@ class PaperForgeStatusView extends ItemView { case 'collection': this._renderCollectionMode(); break; + case 'deep-reading': + this._renderDeepReadingMode(); + break; } // Brief delay before removing switching class for smooth fade transition @@ -1233,6 +1412,19 @@ class PaperForgeStatusView extends ItemView { this._headerTitle.setText('Collection'); modeName = this._currentDomain || 'Unknown Domain'; break; + + case 'deep-reading': + badge.addClass('deep-reading'); + badge.setText('Deep'); + this._headerTitle.setText('Deep Reading'); + if (this._currentPaperEntry && this._currentPaperEntry.title) { + modeName = this._currentPaperEntry.title; + } else if (this._currentPaperKey) { + modeName = this._currentPaperKey; + } else { + modeName = 'Unknown paper'; + } + break; } if (modeName) { diff --git a/paperforge/plugin/styles.css b/paperforge/plugin/styles.css index 2a149b5e..14d97f3d 100644 --- a/paperforge/plugin/styles.css +++ b/paperforge/plugin/styles.css @@ -1040,12 +1040,10 @@ } /* Color variants for bar fills based on lifecycle stage */ -.paperforge-bar-chart .bar-fill.stage-imported { background: var(--color-cyan); } .paperforge-bar-chart .bar-fill.stage-indexed { background: var(--color-blue); } .paperforge-bar-chart .bar-fill.stage-pdf-ready { background: var(--color-purple); } .paperforge-bar-chart .bar-fill.stage-fulltext-ready { background: var(--color-green); } .paperforge-bar-chart .bar-fill.stage-deep-read { background: var(--color-yellow); } -.paperforge-bar-chart .bar-fill.stage-ai-ready { background: var(--color-red); } /* ========================================================================== SECTION 13 — Mode-Aware Content Area @@ -1323,3 +1321,139 @@ .paperforge-collection-health-counts .fail { color: var(--color-red); } + +/* ========================================================================== + SECTION 33 — Deep-Reading Dashboard Mode + ========================================================================== */ +.paperforge-mode-deepreading { + display: flex; + flex-direction: column; + gap: var(--size-4-4, 16px); +} + +.paperforge-deepreading-card { + background: var(--background-primary-alt); + border-radius: var(--radius-m, 8px); + padding: var(--size-4-4, 16px); +} + +.paperforge-deepreading-card-title { + font-weight: 600; + font-size: var(--font-ui-medium); + margin-bottom: var(--size-4-2, 8px); +} + +.paperforge-deepreading-card-header.collapsible { + cursor: pointer; + user-select: none; +} + +.paperforge-deepreading-card-header:hover { + opacity: 0.8; +} + +.paperforge-deepreading-card-body.collapsed { + display: none; +} + +.paperforge-deepreading-status-list { + display: flex; + flex-direction: column; +} + +.paperforge-deepreading-status-row { + display: flex; + justify-content: space-between; + padding: var(--size-4-1, 4px) 0; + border-bottom: 1px solid var(--background-modifier-border); +} + +.paperforge-deepreading-status-row:last-child { + border-bottom: none; +} + +.paperforge-deepreading-status-label { + color: var(--text-muted); + font-size: var(--font-ui-small); +} + +.paperforge-deepreading-status-value { + font-size: var(--font-ui-small); +} + +.paperforge-deepreading-pass1-content { + line-height: 1.6; +} + +.paperforge-deepreading-pass1-subheading { + margin: var(--size-4-2, 8px) 0 var(--size-4-1, 4px); + color: var(--text-accent); +} + +.paperforge-deepreading-pass1-marker { + font-weight: 600; + color: var(--text-normal); + margin: var(--size-4-2, 8px) 0; +} + +.paperforge-deepreading-pass1-text { + color: var(--text-muted); + margin: var(--size-4-1, 4px) 0; +} + +.paperforge-deepreading-empty { + color: var(--text-faint); + font-size: var(--font-ui-small); + padding: var(--size-4-2, 8px); + text-align: center; +} + +/* Session grouping */ +.paperforge-deepreading-session { + margin-bottom: var(--size-4-3, 12px); +} + +.paperforge-deepreading-session-header { + display: flex; + justify-content: space-between; + font-size: var(--font-ui-small); + color: var(--text-faint); + margin-bottom: var(--size-4-2, 8px); + padding-bottom: var(--size-4-1, 4px); + border-bottom: 1px dashed var(--background-modifier-border); +} + +.paperforge-deepreading-session-date { + color: var(--text-muted); +} + +/* Dialog bubbles */ +.paperforge-deepreading-bubble { + padding: var(--size-4-2, 8px) var(--size-4-3, 12px); + border-radius: var(--radius-s, 6px); + margin-bottom: var(--size-4-2, 8px); + max-width: 90%; +} + +.paperforge-deepreading-bubble.question { + background: var(--interactive-accent); + color: var(--text-on-accent); + margin-right: auto; +} + +.paperforge-deepreading-bubble.answer { + background: var(--background-secondary-alt); + color: var(--text-normal); + margin-left: auto; +} + +.paperforge-deepreading-bubble .bubble-label { + font-size: var(--font-smallest, 10px); + opacity: 0.7; + margin-bottom: 2px; +} + +.paperforge-deepreading-bubble .bubble-text { + font-size: var(--font-ui-small); + line-height: 1.5; +} diff --git a/paperforge/worker/asset_index.py b/paperforge/worker/asset_index.py index 08a2a150..7912c8d9 100644 --- a/paperforge/worker/asset_index.py +++ b/paperforge/worker/asset_index.py @@ -26,13 +26,14 @@ from __future__ import annotations import json import logging import os +import shutil import tempfile -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path import filelock -import shutil +from paperforge import __version__ as _paperforge_version from paperforge.config import paperforge_paths logger = logging.getLogger(__name__) @@ -82,6 +83,7 @@ def build_envelope(items: list[dict]) -> dict: "schema_version": "2", "generated_at": "2026-05-04T12:34:56Z", "paper_count": len(items), + "paperforge_version": "1.8.0", "items": items, } """ @@ -89,6 +91,7 @@ def build_envelope(items: list[dict]) -> dict: "schema_version": CURRENT_SCHEMA_VERSION, "generated_at": datetime.now(timezone(timedelta(hours=8))).isoformat(), "paper_count": len(items), + "paperforge_version": _paperforge_version, "items": items, } @@ -158,7 +161,7 @@ def read_index(vault: Path) -> dict | list | None: if not path.exists(): return None try: - with open(path, "r", encoding="utf-8") as fh: + with open(path, encoding="utf-8") as fh: return json.load(fh) except (json.JSONDecodeError, OSError) as exc: logger.warning("Corrupt index at %s, treating as missing: %s", path, exc) @@ -220,8 +223,14 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: Lazy imports inside avoid circular dependencies with ``sync.py``. """ # Lazy imports to avoid circular deps with sync.py + from paperforge.worker._utils import read_json, slugify_filename, write_json + from paperforge.worker.asset_state import ( + compute_health, + compute_lifecycle, + compute_maturity, + compute_next_step, + ) from paperforge.worker.ocr import validate_ocr_meta - from paperforge.worker._utils import read_json, write_json, slugify_filename from paperforge.worker.sync import ( collection_fields, frontmatter_note, @@ -229,12 +238,6 @@ def _build_entry(item: dict, vault: Path, paths: dict, domain: str, zotero_dir: obsidian_wikilink_for_path, obsidian_wikilink_for_pdf, ) - from paperforge.worker.asset_state import ( - compute_lifecycle, - compute_health, - compute_maturity, - compute_next_step, - ) key = item["key"] collection_meta = collection_fields(item.get("collections", [])) @@ -352,7 +355,7 @@ def build_index(vault: Path, verbose: bool = False) -> int: """ # Lazy imports to avoid circular dependencies with sync.py from paperforge.config import load_vault_config - from paperforge.worker._utils import pipeline_paths, read_json # noqa: F811 + from paperforge.worker._utils import pipeline_paths # noqa: F811 from paperforge.worker.base_views import ensure_base_views from paperforge.worker.sync import load_domain_config, load_export_rows diff --git a/tests/sandbox/batch_check.py b/tests/sandbox/batch_check.py new file mode 100644 index 00000000..0c8d4d5b --- /dev/null +++ b/tests/sandbox/batch_check.py @@ -0,0 +1,121 @@ +"""Batch re-process all OCR papers and scan for common defects.""" +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, r"D:\L\Med\Research\99_System\LiteraturePipeline\github-release") +from paperforge.worker.ocr import postprocess_ocr_result + +VAULT = Path(r"D:\L\Med\Research_LitControl_Sandbox") +OCR_DIR = VAULT / "System" / "PaperForge" / "ocr" + +# Collect all papers with result.json +papers = sorted([d.name for d in OCR_DIR.iterdir() if (d / "json" / "result.json").exists()]) +print(f"Processing {len(papers)} papers...\n") + +results = [] +for key in papers: + ocr_root = OCR_DIR / key + json_path = ocr_root / "json" / "result.json" + all_results = json.loads(json_path.read_text(encoding="utf-8")) + + page_count, _, _, error = postprocess_ocr_result(VAULT, key, all_results) + + text = (ocr_root / "fulltext.md").read_text(encoding="utf-8") + lines = text.splitlines() + + defects = [] + + # 1. Image count + images = [l for l in lines if l.startswith("![[")] + img_count = len(images) + + # 2. Bare < outside math (HTML bleed indicator) + text_no_math = re.sub(r'\$[^$]+\$', '', text) + text_no_math = re.sub(r'<!--.*?-->', '', text_no_math) + text_no_math = re.sub(r'<table.*?</table>', '', text_no_math, flags=re.DOTALL) + bare_lt_lines = set() + for i, l in enumerate(text_no_math.splitlines(), 1): + if '<' in l and any(c.isalpha() for c in l): + bare_lt_lines.add(i) + if bare_lt_lines: + defects.append(f"bare < at {sorted(bare_lt_lines)[:3]}") + + # 3. [^correspondence] in captions + corr_in_caption = sum(1 for l in lines if '[^correspondence]' in l and 'Figure' in l) + if corr_in_caption: + defects.append(f"[^correspondence] in {corr_in_caption} captions") + + # 4. *p bleed (bare *p NOT in math) + bare_star = sum(1 for l in lines if re.search(r'(?<!\$)\*\s*p\s*<', l)) + if bare_star: + defects.append(f"bare *p< in {bare_star} lines") + + # 5. Triple-asterisk strong in *p context (e.g. "***") + strong_p = sum(1 for l in lines if '**p' in l and re.search(r'\*{3,}', l)) + if strong_p: + defects.append(f"strong **p potential in {strong_p} lines") + + # 6. Anand split + anand = sum(1 for l in lines if 'An and ' in l and 'Thirupathi' in lines[lines.index(l)] if False) + # simpler check + anand_any = sum(1 for l in lines if ' An and ' in l or 'An and T' in l) + if anand_any: + defects.append("Anand split detected") + + # 7. Orphan captions (Figure N with no preceding image) + fig_caps = [i for i, l in enumerate(lines) if re.match(r'^Figure \d+', l)] + orphan_caps = 0 + for idx in fig_caps: + # Check previous non-empty line before this caption + prev = None + for j in range(idx - 1, -1, -1): + if lines[j].strip() and not lines[j].startswith("<!--"): + prev = lines[j] + break + if prev and not prev.startswith("![["): + orphan_caps += 1 + if orphan_caps: + defects.append(f"{orphan_caps} captions have no preceding image") + + # 8. Suspicious merged images (very wide + tall single image) + found_suspicious = 0 + for l in lines: + if l.startswith("![[") and "images/blocks" in l: + m = re.search(r'(\d+)_(\d+)_(\d+)_(\d+)\.jpg', l) + if m: + w = int(m.group(3)) - int(m.group(1)) + h = int(m.group(4)) - int(m.group(2)) + if w > 700 and h > 700: + found_suspicious += 1 + if found_suspicious: + defects.append(f"{found_suspicious} large images (>700x700)") + + # 9. Cross-page hyphenation + for i, l in enumerate(lines): + stripped = l.strip() + if stripped.endswith('-') and len(stripped) > 3 and i + 1 < len(lines): + next_l = lines[i+1].strip() + if next_l.startswith(('a','b','c','d','e','f','g','h','i','j','k','l','m', + 'n','o','p','q','r','s','t','u','v','w','x','y','z')): + defects.append("cross-page hyphenation") + break + + # 10. LaTeX spacing + loose_dollar = sum(1 for l in lines if '$ ' in l or ' $' in l) + if loose_dollar > 0.5 * len(lines): + defects.append(f"many loose $ ({loose_dollar})") + + verdict = "CLEAN" if not defects else "; ".join(defects) + results.append((key, img_count, page_count, verdict, len(lines))) + +# Summary +clean = sum(1 for _, _, _, v, _ in results if v == "CLEAN") +print(f"Processed {len(papers)} papers: {clean} clean, {len(papers)-clean} with issues\n") + +for key, imgs, pages, verdict, lines in results: + if verdict != "CLEAN": + print(f" {key} ({pages}p, {imgs}img): {verdict}") + +print("\nDone.") diff --git a/tests/sandbox/deep_inspect.py b/tests/sandbox/deep_inspect.py new file mode 100644 index 00000000..766d0c87 --- /dev/null +++ b/tests/sandbox/deep_inspect.py @@ -0,0 +1,42 @@ +"""Detailed inspection of flagged papers.""" +import re +from pathlib import Path + +OCR_DIR = Path(r"D:\L\Med\Research_LitControl_Sandbox\System\PaperForge\ocr") +KEYS = ["2AGGSMVQ", "2BFG5P6B", "2BYRLKQS", "2E4EPHN2", "2GN9LMCW", "2H8MZ27H"] + +for key in KEYS: + md = OCR_DIR / key / "fulltext.md" + text = md.read_text(encoding="utf-8") + lines = text.splitlines() + + print(f"=== {key} ({len(lines)} lines) ===") + + # List all Figure captions and preceding image + fig_lines = [] + for i, l in enumerate(lines): + if re.match(r'^Figure \d+', l): + # Find preceding non-comment, non-blank line + prev = None + for j in range(i-1, -1, -1): + s = lines[j].strip() + if s and not s.startswith("<!--"): + prev = (j, s) + break + before = "IMAGE" if prev and prev[1].startswith("![[") else (prev[1][:80] if prev else "NONE") + fig_lines.append((i+1, l[:100], before)) + + for ln, caption, before in fig_lines: + print(f" L{ln:4d}: {before:30s} | {caption}") + + # Check bare *p< patterns — show actual lines + for i, l in enumerate(lines): + m = re.search(r'(?<!\$)\*\s*p\s*<', l) + if m: + # Find context around match + start = max(0, m.start() - 20) + end = min(len(l), m.end() + 20) + ctx = l[start:end] + print(f" *p at L{i+1}: ...{ctx}...") + + print() diff --git a/tests/sandbox/precise_batch.py b/tests/sandbox/precise_batch.py new file mode 100644 index 00000000..473c35eb --- /dev/null +++ b/tests/sandbox/precise_batch.py @@ -0,0 +1,115 @@ +"""Precise batch defect report.""" +import re +import json +import sys +from pathlib import Path + +sys.path.insert(0, r"D:\L\Med\Research\99_System\LiteraturePipeline\github-release") + +VAULT = Path(r"D:\L\Med\Research_LitControl_Sandbox") +OCR_DIR = VAULT / "System" / "PaperForge" / "ocr" +papers = sorted([d.name for d in OCR_DIR.iterdir() if (d / "json" / "result.json").exists()]) + +print(f"=== BIG BATCH REPORT === ({len(papers)} papers)\n") + +for key in papers: + md = OCR_DIR / key / "fulltext.md" + text = md.read_text(encoding="utf-8") + lines = text.splitlines() + + issues = [] + + # Image count vs caption count + images = [l for l in lines if l.startswith("![[")] + fig_caps = [(i, l) for i, l in enumerate(lines) if re.match(r'^Figure \d+', l)] + img_num = len(images) + cap_num = len(fig_caps) + + # Captions without preceding image (REAL issue) + orphan = 0 + for idx, _ in fig_caps: + prev = None + for j in range(idx-1, -1, -1): + s = lines[j].strip() + if s and not s.startswith("<!--"): + prev = (j, s) + break + if not (prev and prev[1].startswith("![[") and "images/blocks" in prev[1]): + orphan += 1 + if orphan: + issues.append(f"{orphan}/{cap_num} captions lack preceding image") + + # Check: is there an image within 5 lines above each caption? + remote = 0 + for idx, _ in fig_caps: + found = False + for j in range(idx-1, max(-1, idx-6), -1): + if lines[j].strip().startswith("![["): + found = True + break + if not found: + remote += 1 + if remote and remote < cap_num: + issues.append(f"{remote}/{cap_num} captions have image >5 lines away") + + # Large images that could be merged figures + huge_merges = 0 + for l in lines: + m = re.search(r'(\d+)_(\d+)_(\d+)_(\d+)\.jpg', l) + if m: + w = int(m.group(3)) - int(m.group(1)) + h = int(m.group(4)) - int(m.group(2)) + if w > 600 and h > 1000: + huge_merges += 1 + if huge_merges: + issues.append(f"{huge_merges} extremely tall images (>1000px)") + + # Bare < in text lines (not math, not table, not comment) — REAL ISSUE + bare_lt = 0 + for i, l in enumerate(lines): + s = l.strip() + if s.startswith("<!--") or s.startswith("<table") or s.startswith("</table"): + continue + no_math = re.sub(r'\$[^$]+\$', '', l) + if '<' in no_math and any(c.isalpha() for c in no_math): + bare_lt += 1 + if bare_lt: + issues.append(f"{bare_lt} bare < in text (may trigger HTML)") + + # [^correspondence] bleeding + corr = sum(1 for l in lines if '[^correspondence]' in l and 'Figure' in l) + if corr: + issues.append(f"{corr} [^correspondence] in Figure captions") + + # *p bleeding — only check lines where NOT preceded by $ anywhere on the line + star_issues = 0 + for l in lines: + no_math = re.sub(r'\$[^$]+\$', '', l) + if re.search(r'\*\s*p\s*<', no_math): + star_issues += 1 + if star_issues: + issues.append(f"{star_issues} *p< outside math") + + # Anand + if any('An and' in l for l in lines): + issues.append("Anand split found") + + # Text ordering: check for obviously broken headers + for l in lines[:20]: + if l.strip().startswith("### ") and l.strip().count("###") > 2: + issues.append("broken heading") + break + + # Cross-page hyphen + for i, l in enumerate(lines): + s = l.strip() + if s.endswith('-') and len(s) > 3: + prev_words = s.rstrip('-') + if prev_words and not prev_words.endswith('\\') and not prev_words.rstrip('-')[0].isupper(): + issues.append("possible cross-page hyphen") + break + + verdict = ", ".join(issues) if issues else "CLEAN" + print(f" {key}: {img_num}i/{cap_num}c | {verdict}") + +print(f"\nDone.") diff --git a/tests/test_ocr_rendering.py b/tests/test_ocr_rendering.py new file mode 100644 index 00000000..f7e558d3 --- /dev/null +++ b/tests/test_ocr_rendering.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +from PIL import Image + + +def test_caption_group_assignments_respects_columns() -> None: + from paperforge.worker.ocr import caption_group_assignments + + blocks = [ + { + "block_id": 1, + "block_label": "chart", + "block_bbox": [80, 116, 546, 434], + "block_content": "", + }, + { + "block_id": 2, + "block_label": "figure_title", + "block_bbox": [66, 446, 559, 628], + "block_content": "Figure 1. Left column figure.", + }, + { + "block_id": 3, + "block_label": "chart", + "block_bbox": [598, 114, 1063, 493], + "block_content": "", + }, + { + "block_id": 4, + "block_label": "figure_title", + "block_bbox": [584, 503, 1079, 744], + "block_content": "Figure 2. Right column figure.", + }, + ] + + figure_map, _table_map = caption_group_assignments(blocks) + + left_ids = [item["block_id"] for item in figure_map[2]] + right_ids = [item["block_id"] for item in figure_map[4]] + + assert left_ids == [1] + assert right_ids == [3] + + +def test_render_page_blocks_links_media_for_text_caption(tmp_path: Path) -> None: + from paperforge.worker.ocr import render_page_blocks + + vault = tmp_path / "vault" + images_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "images" + page_cache_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "pages" + images_dir.mkdir(parents=True) + page_cache_dir.mkdir(parents=True) + + page_image = page_cache_dir / "page_009.png" + Image.new("RGB", (1200, 1600), color="white").save(page_image) + + result = { + "prunedResult": { + "width": 1200, + "height": 1600, + "parsing_res_list": [ + { + "block_id": 1, + "block_label": "vision_footnote", + "block_bbox": [449, 168, 738, 219], + "block_content": "No Electrical Stimulation\nElectrical Stimulation 100 mV/mm", + }, + { + "block_id": 2, + "block_label": "chart", + "block_bbox": [429, 237, 733, 485], + "block_content": "", + }, + { + "block_id": 3, + "block_label": "chart", + "block_bbox": [772, 238, 1071, 484], + "block_content": "", + }, + { + "block_id": 4, + "block_label": "chart", + "block_bbox": [363, 504, 742, 757], + "block_content": "", + }, + { + "block_id": 5, + "block_label": "chart", + "block_bbox": [766, 503, 1075, 750], + "block_content": "", + }, + { + "block_id": 6, + "block_label": "chart", + "block_bbox": [428, 774, 729, 1016], + "block_content": "", + }, + { + "block_id": 7, + "block_label": "chart", + "block_bbox": [765, 768, 1075, 1013], + "block_content": "", + }, + { + "block_id": 8, + "block_label": "figure_title", + "block_bbox": [374, 1046, 1143, 1077], + "block_content": "Days post culture in osteogenic differentiation supplemented medium", + }, + { + "block_id": 9, + "block_label": "text", + "block_bbox": [373, 1101, 1143, 1258], + "block_content": "Figure 4 RT-qPCR results. Temporal changes in messenger RNA (mRNA) of (A) Runx2, (B) Osteopontin and (C) Col1A2.", + }, + ], + }, + "inputImage": "", + } + + with patch("paperforge.worker.ocr.render_pdf_page_cached", return_value=page_image): + rendered = render_page_blocks(vault, 9, result, images_dir, page_cache_dir, pdf_doc=None) + + assert any(line.startswith("![[") for line in rendered) + assert any(line.startswith("Figure 4 RT-qPCR results.") for line in rendered)