From 8335bce9925e70d5e207ddff485277d25e7b08fb Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Tue, 12 May 2026 18:11:37 +0800 Subject: [PATCH] bump: 1.5.5 -> 1.5.6rc1 (Memory Layer RC1) --- .github/workflows/ci-chaos.yml | 37 + .github/workflows/ci.yml | 114 ++ .github/workflows/release.yml | 42 + .../2026-05-12-memory-layer-REVIEW-v3.md | 101 ++ .../plans/2026-05-12-memory-layer-REVIEW.md | 377 +++++ .../plans/2026-05-12-memory-layer.md | 1235 +++++++++++++++++ .../specs/2026-05-12-memory-layer-design.md | 279 ++++ manifest.json | 2 +- paperforge/__init__.py | 2 +- paperforge/plugin/manifest.json | 2 +- 10 files changed, 2188 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci-chaos.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 docs/superpowers/plans/2026-05-12-memory-layer-REVIEW-v3.md create mode 100644 docs/superpowers/plans/2026-05-12-memory-layer-REVIEW.md create mode 100644 docs/superpowers/plans/2026-05-12-memory-layer.md create mode 100644 docs/superpowers/specs/2026-05-12-memory-layer-design.md diff --git a/.github/workflows/ci-chaos.yml b/.github/workflows/ci-chaos.yml new file mode 100644 index 00000000..5eb83a02 --- /dev/null +++ b/.github/workflows/ci-chaos.yml @@ -0,0 +1,37 @@ +name: Chaos Tests (L6) + +on: + schedule: + # Weekly: Sunday 06:00 UTC + - cron: "0 6 * * 0" + workflow_dispatch: + # Manual trigger from GitHub UI + +jobs: + chaos-tests: + name: Chaos / Destructive Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install package with test dependencies + run: | + pip install -e ".[test]" + pip install pytest pytest-timeout responses PyYAML + + - name: Run chaos tests + run: | + python -m pytest tests/chaos/ -m chaos -v --tb=long --timeout=120 \ + --junit-xml=chaos-results.xml + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: chaos-test-results + path: chaos-results.xml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..0dccc57d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,114 @@ +# CI — Simplified Pipeline +# Runs on push/PR to master. All tests run (no -x early exit). + +name: CI + +on: + push: + branches: [main, master] + paths-ignore: + - "**.md" + - "docs/**" + pull_request: + branches: [main, master] + +env: + PYTHONIOENCODING: utf-8 + +jobs: + # --------------------------------------------------------------------------- + # L0 — Version consistency check + # --------------------------------------------------------------------------- + version-check: + name: L0 — Version Sync + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install package + run: pip install -e . + - name: Check version consistency + run: python scripts/check_version_sync.py + + # --------------------------------------------------------------------------- + # L1 — Unit tests (3 OS x 1 Python) + # --------------------------------------------------------------------------- + unit-tests: + name: L1 — Unit Tests (${{ matrix.os }}, py3.11) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install package with test deps + run: pip install -e ".[test]" + - name: Run unit tests + shell: bash + run: | + python -m pytest tests/ \ + --ignore=tests/sandbox \ + --ignore=tests/cli \ + --ignore=tests/e2e \ + --ignore=tests/journey \ + --ignore=tests/chaos \ + --ignore=tests/audit \ + -v --tb=short --timeout=60 + + # --------------------------------------------------------------------------- + # L3 — Plugin tests (Vitest) + # --------------------------------------------------------------------------- + plugin-tests: + name: L3 — Plugin Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: paperforge/plugin/package-lock.json + - run: npm ci + working-directory: paperforge/plugin + - run: npx vitest run --reporter=verbose + working-directory: paperforge/plugin + + # --------------------------------------------------------------------------- + # L4 — E2E + Audit + # --------------------------------------------------------------------------- + e2e-tests: + name: L4 — E2E + Audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install package with test deps + run: pip install -e ".[test]" + - name: Run E2E tests + run: python -m pytest tests/e2e/ -m e2e -v --tb=short --timeout=120 + - name: Run audit tests + run: python -m pytest tests/audit/ -m audit -v --tb=short --timeout=120 + + # --------------------------------------------------------------------------- + # Merge gate + # --------------------------------------------------------------------------- + alls-green: + name: All Checks Passed + if: always() + needs: + - unit-tests + - plugin-tests + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@v1.2.2 + with: + allowed-skips: version-check + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..c8aab79c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,42 @@ +# Auto-release Obsidian plugin on tag push +# Triggered by tags like v1.4.18. Runs plugin tests, then creates a GitHub Release +# with the 4 required Obsidian plugin files. + +name: Release + +on: + push: + tags: + - "v*" + +jobs: + release: + name: Release Plugin + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: paperforge/plugin/package-lock.json + + - name: Run plugin tests + working-directory: paperforge/plugin + run: | + npm ci + npx vitest run --reporter=verbose + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + name: ${{ github.ref_name }} + generate_release_notes: true + files: | + paperforge/plugin/main.js + paperforge/plugin/styles.css + paperforge/plugin/manifest.json + paperforge/plugin/versions.json diff --git a/docs/superpowers/plans/2026-05-12-memory-layer-REVIEW-v3.md b/docs/superpowers/plans/2026-05-12-memory-layer-REVIEW-v3.md new file mode 100644 index 00000000..5b2cd7b4 --- /dev/null +++ b/docs/superpowers/plans/2026-05-12-memory-layer-REVIEW-v3.md @@ -0,0 +1,101 @@ +--- +phase: memory-layer-plan-v3-quick-check +reviewed: 2026-05-12T09:25:08Z +depth: standard +files_reviewed: 1 +files_reviewed_list: + - docs/superpowers/plans/2026-05-12-memory-layer.md +findings: + critical: 0 + warning: 1 + info: 0 + total: 1 +status: issues_found +--- + +# Phase: Memory Layer Plan v3 Quick Check + +**Reviewed:** 2026-05-12T09:25:08Z +**Depth:** standard (plan-only, cross-referenced against codebase for `--key` validation) +**Files Reviewed:** 1 +**Status:** ISSUES_FOUND (1 WARNING remaining) + +--- + +## Summary + +Quick final check of the implementation plan after v3 review fixes. All 5 named issues from the prior review are **confirmed fixed**. The plan incorporates all 14 fixes from the original v1 deep review (5 CR + 5 WR + 4 IN). One new WARNING-level issue identified in the `_entry_from_row` function. + +--- + +## Named Issue Verification + +| Issue | Status | Evidence | +|-------|--------|----------| +| **N-BLKR-01**: hash query inside try block | **FIXED** | Lines 713-716 — `stored_hash_row = conn.execute(...)` is inside the `try:` block at line 708 | +| **N-BLKR-02**: NameError on `status` in memory.py | **FIXED** | Line 985 — `if result.ok:` guards access to `status` on line 986. `result` is always assigned in both try/except branches | +| **N-WRN-01**: paper_status empty fields for unresolved | **FIXED** | Line 1055 — `if data.get("resolved"):` guards detailed field printing | +| **N-INFO-01**: private `_compute_hash` renamed to `compute_hash` | **FIXED** | Line 451 — `def compute_hash(...)` (public). Line 683 — `from paperforge.memory.builder import compute_hash` | +| **N-INFO-02**: JSON decode logged with `logging.warning` | **FIXED** | Lines 762-764 — `logging.warning("Corrupted JSON in column %s for paper %s", key, ...)` | + +All 5 named issues from the prior review are resolved in the plan. + +--- + +## Original v1 Review Issue Verification (bonus) + +Cross-checked all 14 issues from `2026-05-12-memory-layer-REVIEW.md`: + +| Issue | Status | +|-------|--------| +| CR-01: `make_result` import | **FIXED** — line 910 imports only `PFError, PFResult` | +| CR-02: hash not checked | **FIXED** — lines 713-746 compare stored hash vs computed | +| CR-03: legacy format crash in builder | **FIXED** — lines 475-480 handle `isinstance(envelope, list)` | +| CR-04: legacy format crash in query | **FIXED** — lines 725-733 handle bare list | +| CR-05: Windows-path URI bug | **FIXED** — line 122 uses `db_path.as_posix()` | +| WR-01: `--force` flag | **FIXED** — removed from CLI parser (lines 1080-1082) | +| WR-02: ambiguous query returns full status | **FIXED** — lines 823-838 return candidates only when >1 | +| WR-03: recommended_action missing | **FIXED** — lines 846-855 compute concrete action strings | +| WR-04: zero test coverage | **REMAINS** — plan still has only 4 schema + 3 hash tests | +| WR-05: CLI dispatch pattern | **FIXED** — lines 1089-1099 use simple dispatch | +| IN-01: unused compute_health import | **FIXED** — removed from builder imports (lines 417-422) | +| IN-02: _COMMAND_REGISTRY not consumed | **REMAINS** — still present but rate-limited to INFO | +| IN-03: compute_hash .get vs direct | **FIXED** — line 452 uses `e["zotero_key"]` (direct access) | +| IN-04: fragile rstrip("_json") | **FIXED** — line 760 uses `key[:-5]` instead of `rstrip` | + +--- + +## Warnings + +### WR-V3-01: Data silently lost when JSON decode fails in `_entry_from_row` + +**File:** `docs/superpowers/plans/2026-05-12-memory-layer.md:759-760` +**Issue:** When `json.loads()` raises `JSONDecodeError`, `entry.pop(key)` has already executed — the original `_json` column value is removed from the result dict and never restored. The field disappears silently from query output. + +```python +# Current (plan line 759-760) +try: + entry[key[:-5]] = json.loads(entry.pop(key)) # pop() happens BEFORE json.loads() +except json.JSONDecodeError: + logging.warning(...) # original value already lost +``` + +**Fix:** +```python +# Pop first, then try to decode, restore on failure +raw = entry.pop(key) +try: + entry[key[:-5]] = json.loads(raw) +except json.JSONDecodeError: + entry[key] = raw # keep original JSON string visible + logging.warning( + "Corrupted JSON in column %s for paper %s", + key, entry.get("zotero_key", "?"), + ) +``` + +--- + +_Reviewed: 2026-05-12T09:25:08Z_ +_Reviewer: VT-OS/OPENCODE (gsd-code-reviewer)_ +_Depth: standard_ diff --git a/docs/superpowers/plans/2026-05-12-memory-layer-REVIEW.md b/docs/superpowers/plans/2026-05-12-memory-layer-REVIEW.md new file mode 100644 index 00000000..39c62698 --- /dev/null +++ b/docs/superpowers/plans/2026-05-12-memory-layer-REVIEW.md @@ -0,0 +1,377 @@ +--- +phase: memory-layer-plan-review +reviewed: 2026-05-12T18:30:00Z +depth: deep +files_reviewed: 9 +files_reviewed_list: + - docs/superpowers/plans/2026-05-12-memory-layer.md + - docs/superpowers/specs/2026-05-12-memory-layer-design.md + - paperforge/config.py + - paperforge/cli.py + - paperforge/commands/__init__.py + - paperforge/core/result.py + - paperforge/core/errors.py + - paperforge/worker/asset_state.py + - paperforge/worker/asset_index.py +findings: + critical: 5 + warning: 5 + info: 4 + total: 14 +status: issues_found +--- + +# Phase: Memory Layer Plan Review + +**Reviewed:** 2026-05-12T18:30:00Z +**Depth:** deep (cross-file analysis with import graph tracing) +**Files Reviewed:** 9 +**Status:** ISSUES_FOUND + +## Verdict: ISSUES_FOUND + +5 BLOCKER, 5 WARNING, 4 INFO issues detected. Plan must not be executed until BLOCKER items are resolved. + +--- + +## Summary + +The plan maps spec requirements to tasks with reasonable granularity, and the overall architecture (SQLite under `paperforge/memory/`, derived from `formal-library.json`, PFResult-enveloped CLI) is sound. However, the cross-file trace against the actual codebase reveals **five BLOCKER defects** — a non-existent import, a missing spec-critical hash check, two crash-on-legacy-format scenarios, and a Windows-path URI bug. Five WARNING-level issues include an unimplemented `--force` flag, a behavioral divergence from spec for ambiguous queries, a missing `recommended_action` field, near-zero test coverage for business logic, and an inconsistent CLI dispatch pattern. + +--- + +## Critical Issues + +### CR-01: Import of non-existent `make_result` in `memory.py` + +**File:** Plan Task 6, Step 1 (`paperforge/commands/memory.py` line 4) +**Issue:** The plan code imports `make_result` from `paperforge.core.result`: +```python +from paperforge.core.result import PFError, PFResult, make_result +``` +`make_result` is **not defined anywhere** in the codebase. Verified by grep of the entire `paperforge/` tree — zero matches. `core/result.py` (lines 1-79) exports only `PFError` and `PFResult`. This would cause `ImportError` at runtime on every invocation of `paperforge memory`. + +**Fix:** +```python +# Remove make_result from the import line — it's never used in the function body either. +from paperforge.core.result import PFError, PFResult +``` + +--- + +### CR-02: `get_memory_status` does not check `canonical_index_hash` + +**File:** Plan Task 5, Step 1 (`paperforge/memory/query.py`, `get_memory_status()`) +**Issue:** The spec (Design Spec lines 221-226) explicitly requires `memory status` to verify `canonical_index_hash` against the SHA-256 of the current `formal-library.json`: +> - `canonical_index_hash` matches computed hash of current `formal-library.json` → `fresh: bool` + +The plan's implementation (lines 678-717) computes `fresh` as only: +```python +result["fresh"] = result["schema_ok"] and result["count_match"] +``` +The `canonical_index_hash` stored in `meta` during build is never read back and never compared. The status command will report `fresh: true` even when the canonical index has changed since the last build — giving a falsely green "fresh" signal that causes stale paper-status results. + +**Fix:** In `get_memory_status()` after the read-only connection is opened, add: +```python +# Read stored hash from meta +stored_hash_row = conn.execute( + "SELECT value FROM meta WHERE key = 'canonical_index_hash'" +).fetchone() +stored_hash = stored_hash_row["value"] if stored_hash_row else "" + +# Recompute hash from current index +envelope = read_index(vault) +items = envelope.get("items", []) if isinstance(envelope, dict) else [] +from paperforge.memory.builder import _compute_hash +current_hash = _compute_hash(items) if items else "" + +result["hash_match"] = stored_hash == current_hash +result["fresh"] = result["schema_ok"] and result["count_match"] and result["hash_match"] +``` + +--- + +### CR-03: `build_from_index` crashes on legacy-format (bare list) index + +**File:** Plan Task 4, Step 1 (`paperforge/memory/builder.py`, line 467-471) +**Issue:** `read_index(vault)` in `asset_index.py` (line 160-176) can return a **bare list** (legacy pre-v1.6 format). The `build_from_index` function only checks for `None`: +```python +envelope = read_index(vault) +if envelope is None: + raise FileNotFoundError(...) +items = envelope.get("items", []) # <-- CRASH: list has no .get() +``` +If the vault has a legacy-format `formal-library.json` (not yet migrated by a sync run), `envelope` is a `list`, and `envelope.get(...)` raises `AttributeError`. The existing codebase has `is_legacy_format()` and `migrate_legacy_index()` in `asset_index.py` (lines 178-212) specifically for this case. + +**Fix:** Add legacy format detection after the `None` check: +```python +envelope = read_index(vault) +if envelope is None: + raise FileNotFoundError( + "Canonical index not found. Run paperforge sync --rebuild-index." + ) +from paperforge.worker.asset_index import is_legacy_format +if is_legacy_format(envelope): + raise FileNotFoundError( + "Canonical index is in legacy (bare-list) format. " + "Run paperforge sync --rebuild-index to migrate." + ) +items = envelope.get("items", []) +generated_at = envelope.get("generated_at", "") +``` + +--- + +### CR-04: `get_memory_status` crashes on legacy-format index + +**File:** Plan Task 5, Step 1 (`paperforge/memory/query.py`, line 708-713) +**Issue:** Same legacy-format crash as CR-03, but in the read path: +```python +envelope = read_index(vault) +if envelope: + result["paper_count_index"] = envelope.get("paper_count", 0) # CRASH on list +``` +A bare-list envelope causes `AttributeError`. + +**Fix:** Add the same `is_legacy_format` guard: +```python +envelope = read_index(vault) +if envelope and isinstance(envelope, dict): + result["paper_count_index"] = envelope.get("paper_count", 0) + ... +``` + +--- + +### CR-05: Windows-path URI incompatibility in `get_connection` read-only mode + +**File:** Plan Task 2, Step 2 (`paperforge/memory/db.py`, line 122-123) +**Issue:** +```python +uri = f"file:{db_path}?mode=ro" if read_only else str(db_path) +conn = sqlite3.connect(uri, uri=read_only) +``` +On Windows, `db_path` contains backslashes (e.g., `D:\Vault\System\PaperForge\indexes\paperforge.db`). The constructed URI `file:D:\Vault\...?mode=ro` is NOT a valid [RFC 8089 file URI](https://datatracker.ietf.org/doc/html/rfc8089). SQLite's URI parser requires either `file:///D:/...` (authority path) or `file:D:/...` (local path with forward slashes). With backslashes, `sqlite3.connect(..., uri=True)` may fail with `sqlite3.OperationalError: unable to open database file` or silently misinterpret the path. + +**Fix:** Normalize the path to use forward slashes before constructing the URI: +```python +def get_connection(db_path: Path, read_only: bool = False) -> sqlite3.Connection: + if read_only: + # Windows-safe: convert to forward slashes for SQLite URI parser + posix_path = str(db_path.resolve()).replace("\\", "/") + uri = f"file:{posix_path}?mode=ro" + else: + uri = str(db_path) + conn = sqlite3.connect(uri, uri=read_only) + conn.row_factory = sqlite3.Row + if not read_only: + conn.execute("PRAGMA journal_mode=WAL;") + conn.execute("PRAGMA foreign_keys=ON;") + return conn +``` + +--- + +## Warnings + +### WR-01: `--force` flag on `memory build` defined but never implemented + +**File:** Plan Task 6, Step 3 (cli.py parser) + Task 4 builder +**Issue:** The CLI parser adds `--force` to `memory build`: +```python +p_memory_build.add_argument("--force", action="store_true", help="Force rebuild") +``` +Neither `memory.run()` nor `build_from_index()` checks `args.force`. The builder always deletes all paper data and rebuilds (lines 486-488), making `--force` redundant for the current logic. However, a future optimization that caches unchanged entries would make `--force` meaningful. Either implement the flag or remove it — dead CLI interfaces degrade user experience and create maintenance debt. + +**Fix:** Either (a) remove the `--force` argument entirely from the parser, or (b) wire it through: +```python +# In builder.py: add force parameter +def build_from_index(vault: Path, force: bool = False) -> dict: + ... + if force: + drop_all_tables(conn) + ... +# In memory.py: +counts = build_from_index(vault, force=getattr(args, "force", False)) +``` + +--- + +### WR-02: Ambiguous query (>1 results) returns full status instead of candidate list only + +**File:** Plan Task 5 (`paperforge/memory/query.py`, `get_paper_status()`, lines 775-793) +**Issue:** The spec (Design Spec line 243) states: +> **>1 results:** Candidate list only (no full status details) + +The plan returns full status for the first match PLUS the candidate list: +```python +entry = entries[0] +assets = get_paper_assets(conn, entry["zotero_key"]) +entry["health"] = compute_health(entry) # Full status details computed +entry["candidates"] = entries if len(entries) > 1 else None +entry["assets"] = assets +return entry +``` +And the CLI output (paper_status.py lines 986-991) always prints title/year/lifecycle/next_step — even when multiple candidates exist. This violates the spec's "candidate list only" requirement for ambiguous queries. + +**Fix:** When `len(entries) > 1`, return candidate summary only: +```python +if len(entries) > 1: + return { + "candidates": [ + { + "zotero_key": e.get("zotero_key", ""), + "title": e.get("title", ""), + "year": e.get("year", ""), + "doi": e.get("doi", ""), + "domain": e.get("domain", ""), + } + for e in entries + ], + "candidate_count": len(entries), + } +``` + +--- + +### WR-03: `recommended_action` field missing from paper-status output + +**File:** Plan Task 5 (`paperforge/memory/query.py`, `get_paper_status()`) + Task 6 (`paper_status.py`) +**Issue:** The spec (Design Spec lines 252-253) requires: +> `recommended_action`: e.g., `"/pf-deep ABCDEFG"` or `"paperforge sync"` or `"paperforge ocr"` + +The plan only returns `entry["next_step"]` (e.g., `"/pf-deep"`) in the output but never computes a concrete `recommended_action` like `"/pf-deep ABCDEFG"`. The spec implies this should be a ready-to-use command string with the paper key substituted in. + +**Fix:** In `get_paper_status()`, after computing health, add: +```python +step = entry.get("next_step", "") +zkey = entry.get("zotero_key", "") +action_map = { + "/pf-deep": f"/pf-deep {zkey}", + "ocr": f"paperforge ocr --key {zkey}", + "sync": "paperforge sync", + "repair": "paperforge repair", + "ready": "Ready — no action needed", +} +entry["recommended_action"] = action_map.get(step, step) +``` + +--- + +### WR-04: Core business logic functions have zero test coverage + +**File:** Plan Tasks 4-5 (test files) +**Issue:** The plan specifies 8 tests total: +- 4 schema tests (table creation/deletion/schema version) — good +- 3 builder tests — but ALL three test only `_compute_hash`, a 10-line helper. `build_from_index()` (~150 lines) has **zero tests**. +- 1 query test — only tests `get_memory_status()` with a nonexistent vault path. `lookup_paper()`, `get_paper_assets()`, `get_paper_status()`, and `_entry_from_row()` have **zero tests**. + +Untested edge cases include: empty items list, schema version mismatch trigger, corrupt JSON in authors/collections, exact zotero_key lookup, DOI lookup, title substring search, no-results path, asset reconstruction with None values. + +**Fix:** Add at minimum: +- `test_build_from_index_empty_items()` — ensure handles empty index gracefully +- `test_build_from_index_schema_mismatch()` — verify drop+rebuild on version change +- `test_build_from_index_populates_correctly()` — build from a mock envelope, verify paper count/asset count +- `test_lookup_paper_by_key()` — exact zotero_key match +- `test_lookup_paper_by_doi()` — DOI lookup +- `test_lookup_paper_by_title_substring()` — LIKE match +- `test_lookup_paper_no_results()` — returns empty list +- `test_get_paper_status_returns_none_for_missing()` — paper not found +- `test_entry_from_row_handles_null_fields()` — None values don't crash + +--- + +### WR-05: CLI dispatch pattern inconsistent with existing codebase + +**File:** Plan Task 6, Step 3 (cli.py dispatch blocks) +**Issue:** The plan adds verbose-index carving logic in the dispatch blocks: +```python +if args.command == "memory": + argv = sys.argv.copy() + try: + idx = argv.index("memory") + args.verbose = "--verbose" in argv[idx:] or "-v" in argv[idx:] + except ValueError: + pass + from paperforge.commands import memory + return memory.run(args) +``` +No other command dispatch in `cli.py` (lines 407-533) uses this pattern. All 15 existing command dispatches simply import and call `run(args)`. The `--verbose` flag is already a top-level argument parsed by argparse (cli.py lines 132-136), and `configure_logging(verbose=...)` is called at line 402 BEFORE any dispatch. This carving code is redundant and adds 14 lines of unnecessary complexity per command. + +**Fix:** Follow the existing pattern — just import and dispatch: +```python +if args.command == "memory": + from paperforge.commands import memory + return memory.run(args) + +if args.command == "paper-status": + from paperforge.commands import paper_status + return paper_status.run(args) +``` + +--- + +## Info + +### IN-01: Unused `compute_health` import in `builder.py` + +**File:** Plan Task 4, Step 1 (`paperforge/memory/builder.py`, line 414) +**Issue:** `compute_health` is imported but never called in `build_from_index()`. Per the spec (line 141), health dimensions are computed at query time only, so this import is conceptually correct to exclude. The dead import is harmless but clutters the import block. + +**Fix:** Remove `compute_health` from the builder import: +```python +from paperforge.worker.asset_state import ( + compute_lifecycle, + compute_maturity, + compute_next_step, +) +``` + +--- + +### IN-02: `_COMMAND_REGISTRY` entries not consumed by `cli.py` dispatch + +**File:** Plan Task 6, Step 4 (`paperforge/commands/__init__.py`) +**Issue:** The plan adds `"memory"` and `"paper-status"` to `_COMMAND_REGISTRY`, which powers `get_command_module()` for dynamic dispatch. However, `cli.py` uses hard-coded `if/elif` chains (not `get_command_module()`), so these registry entries are unused by the primary dispatch path. The entries are only consumed if some other code path calls `get_command_module("memory")`. + +**Fix:** Not critical for Phase 1, but either (a) use `get_command_module()` in cli.py dispatch to reduce duplication, or (b) document that the registry exists for future dynamic-dispatch migration. + +--- + +### IN-03: `_compute_hash` uses `.get()` instead of direct key access per spec + +**File:** Plan Task 4, Step 1 (`paperforge/memory/builder.py`, line 448-449) +**Issue:** The spec (line 202) explicitly says: +> `sorted(items, key=lambda e: e["zotero_key"])` + +The plan uses `e.get("zotero_key", "")` — a safe-access variant. This is arguably more robust (it won't crash on malformed entries), but the spec's direct-access was an intentional design choice to fail-loud on corrupt data rather than silently producing a different hash. Decide which contract you want. + +**Fix:** Either align with spec (remove `.get()` for loud failure) or update the spec to accept safe access. + +--- + +### IN-04: `_entry_from_row` uses fragile `.rstrip("_json")` + +**File:** Plan Task 5, Step 1 (`paperforge/memory/query.py`, line 729) +**Issue:** +```python +entry[key.rstrip("_json")] = json.loads(entry.pop(key)) +``` +`rstrip("_json")` removes any trailing characters in the set `{'_', 'j', 's', 'o', 'n'}`, not the literal substring `"_json"`. For `"authors_json"` this produces `"authors"` (correct), and for `"collections_json"` it produces `"collections"` (correct). But if future columns with names like `"version_json"` or `"annotation_json"` were added, this would produce `"versi"` or `"annotati"` — silently wrong. The fix is trivial and prevents future bugs. + +**Fix:** +```python +for key in ("authors_json", "collections_json"): + if key in entry and entry[key]: + try: + clean_key = key[:-5] # strip "_json" suffix (exactly 5 chars) + entry[clean_key] = json.loads(entry.pop(key)) + except json.JSONDecodeError: + pass +``` + +--- + +_Reviewed: 2026-05-12T18:30:00Z_ +_Reviewer: VT-OS/OPENCODE (gsd-code-reviewer)_ +_Depth: deep_ diff --git a/docs/superpowers/plans/2026-05-12-memory-layer.md b/docs/superpowers/plans/2026-05-12-memory-layer.md new file mode 100644 index 00000000..aa03cf54 --- /dev/null +++ b/docs/superpowers/plans/2026-05-12-memory-layer.md @@ -0,0 +1,1235 @@ +# Memory Layer Phase 1 — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) +> or superpowers:executing-plans to implement this plan task-by-task. +> Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a SQLite-backed Memory Layer with `memory build`, `memory status`, and `paper-status` commands. + +**Architecture:** New `paperforge/memory/` package with connection, schema, builder, and query modules. +Commands follow the existing CLI pattern (parser registration + `commands/` module dispatch + PFResult envelope). + +**Tech Stack:** Python stdlib `sqlite3`, `hashlib`, existing `paperforge.core.result`, `paperforge.worker.asset_index`, `paperforge.worker.asset_state`. + +**Spec:** `docs/superpowers/specs/2026-05-12-memory-layer-design.md` + +--- + +## File Structure Map + +``` +Create: + paperforge/memory/__init__.py — package init, re-export key types + paperforge/memory/db.py — get_connection(), get_memory_db_path() + paperforge/memory/schema.py — CURRENT_SCHEMA_VERSION, CREATE TABLE SQL, drop/create tables + paperforge/memory/builder.py — build_from_index() — reads formal-library.json, populates SQLite + paperforge/memory/query.py — lookup_paper(), get_paper_status(), get_memory_status() + paperforge/commands/memory.py — CLI run() for "memory build" and "memory status" + paperforge/commands/paper_status.py — CLI run() for "paper-status" + + tests/unit/memory/__init__.py + tests/unit/memory/test_schema.py + tests/unit/memory/test_builder.py + tests/unit/memory/test_query.py + +Modify: + paperforge/config.py:330-339 — add "memory_db" path key + paperforge/cli.py:258-259 — register "memory" and "paper-status" subcommands + paperforge/commands/__init__.py:4-13 — add to _COMMAND_REGISTRY +``` + +--- + +### Task 1: Register `memory_db` path in config + +**Files:** +- Modify: `paperforge/config.py:330-339` + +- [ ] **Step 1: Add `memory_db` key to `paperforge_paths()` return dict** + +```python +# At paperforge/config.py, after line 338 ("index": ...): +"memory_db": paperforge / "indexes" / "paperforge.db", +``` + +- [ ] **Step 2: Verify** + +```bash +python -c "from paperforge.config import paperforge_paths; p=paperforge_paths(); print(p.get('memory_db'), p.get('index'))" +``` + +Expected: both paths point under `.../PaperForge/indexes/`. + +- [ ] **Step 3: Commit** + +```bash +git add paperforge/config.py +git commit -m "feat(config): add memory_db path key for Memory Layer" +``` + +--- + +### Task 2: `paperforge/memory/__init__.py` and `db.py` + +**Files:** +- Create: `paperforge/memory/__init__.py` +- Create: `paperforge/memory/db.py` +- Test: `tests/unit/memory/test_schema.py` (write later) + +- [ ] **Step 1: Write `__init__.py`** + +```python +from __future__ import annotations + +from paperforge.memory.db import get_connection, get_memory_db_path +from paperforge.memory.schema import ensure_schema, drop_all_tables + +__all__ = [ + "get_connection", + "get_memory_db_path", + "ensure_schema", + "drop_all_tables", +] +``` + +- [ ] **Step 2: Write `db.py`** + +```python +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from paperforge.config import paperforge_paths + + +def get_memory_db_path(vault: Path) -> Path: + """Return the absolute path to paperforge.db.""" + paths = paperforge_paths(vault) + db_path = paths.get("memory_db") + if not db_path: + raise FileNotFoundError("memory_db path not configured") + return db_path + + +def get_connection(db_path: Path, read_only: bool = False) -> sqlite3.Connection: + """Open a SQLite connection to paperforge.db with WAL mode. + + Args: + db_path: Path to paperforge.db. + read_only: If True, open in read-only mode (for queries). + """ + if read_only: + uri = "file:" + db_path.as_posix() + "?mode=ro" + conn = sqlite3.connect(uri, uri=True) + else: + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + if not read_only: + conn.execute("PRAGMA journal_mode=WAL;") + conn.execute("PRAGMA foreign_keys=ON;") + return conn +``` + +- [ ] **Step 3: Run a manual import check** + +```bash +python -c "from paperforge.memory import get_connection, get_memory_db_path; print('OK')" +``` + +- [ ] **Step 4: Commit** + +```bash +git add paperforge/memory/__init__.py paperforge/memory/db.py +git commit -m "feat(memory): add db.py with connection and path resolution" +``` + +--- + +### Task 3: `paperforge/memory/schema.py` + +**Files:** +- Create: `paperforge/memory/schema.py` +- Create: `tests/unit/memory/__init__.py` +- Create: `tests/unit/memory/test_schema.py` + +- [ ] **Step 1: Write `schema.py` with SQL definitions** + +```python +from __future__ import annotations + +import sqlite3 + +CURRENT_SCHEMA_VERSION = 1 + +CREATE_META = """ +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +""" + +CREATE_PAPERS = """ +CREATE TABLE IF NOT EXISTS papers ( + zotero_key TEXT PRIMARY KEY, + citation_key TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + year TEXT, + doi TEXT, + pmid TEXT, + journal TEXT, + first_author TEXT, + authors_json TEXT, + abstract TEXT, + domain TEXT, + collection_path TEXT, + collections_json TEXT, + has_pdf INTEGER NOT NULL DEFAULT 0, + do_ocr INTEGER, + analyze INTEGER, + ocr_status TEXT, + deep_reading_status TEXT, + ocr_job_id TEXT, + impact_factor REAL, + lifecycle TEXT, + maturity_level INTEGER, + maturity_name TEXT, + next_step TEXT, + pdf_path TEXT, + note_path TEXT, + main_note_path TEXT, + paper_root TEXT, + fulltext_path TEXT, + ocr_md_path TEXT, + ocr_json_path TEXT, + ai_path TEXT, + deep_reading_md_path TEXT, + updated_at TEXT +); +""" + +CREATE_ASSETS = """ +CREATE TABLE IF NOT EXISTS paper_assets ( + paper_id TEXT NOT NULL, + asset_type TEXT NOT NULL, + path TEXT NOT NULL, + exists_on_disk INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (paper_id, asset_type), + FOREIGN KEY (paper_id) REFERENCES papers(zotero_key) +); +""" + +CREATE_ALIASES = """ +CREATE TABLE IF NOT EXISTS paper_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + paper_id TEXT NOT NULL, + alias TEXT NOT NULL, + alias_norm TEXT NOT NULL, + alias_type TEXT NOT NULL, + FOREIGN KEY (paper_id) REFERENCES papers(zotero_key) +); +""" + +INDEX_SQL = [ + "CREATE INDEX IF NOT EXISTS idx_papers_doi ON papers(doi);", + "CREATE INDEX IF NOT EXISTS idx_papers_citation_key ON papers(citation_key);", + "CREATE INDEX IF NOT EXISTS idx_papers_domain ON papers(domain);", + "CREATE INDEX IF NOT EXISTS idx_papers_year ON papers(year);", + "CREATE INDEX IF NOT EXISTS idx_papers_ocr_status ON papers(ocr_status);", + "CREATE INDEX IF NOT EXISTS idx_papers_deep_status ON papers(deep_reading_status);", + "CREATE INDEX IF NOT EXISTS idx_papers_lifecycle ON papers(lifecycle);", + "CREATE INDEX IF NOT EXISTS idx_papers_next_step ON papers(next_step);", +] + +ALL_TABLES = ["papers", "paper_assets", "paper_aliases", "meta"] + + +def ensure_schema(conn: sqlite3.Connection) -> None: + """Create tables and indexes if they don't exist.""" + conn.execute(CREATE_META) + conn.execute(CREATE_PAPERS) + conn.execute(CREATE_ASSETS) + conn.execute(CREATE_ALIASES) + for idx_sql in INDEX_SQL: + conn.execute(idx_sql) + conn.commit() + + +def drop_all_tables(conn: sqlite3.Connection) -> None: + """Drop all Memory Layer tables (for rebuild).""" + for table in ALL_TABLES: + conn.execute(f"DROP TABLE IF EXISTS {table};") + conn.commit() + + +def get_schema_version(conn: sqlite3.Connection) -> int: + """Read the stored schema version from meta table, or 0 if not found.""" + try: + row = conn.execute( + "SELECT value FROM meta WHERE key = 'schema_version'" + ).fetchone() + return int(row["value"]) if row else 0 + except sqlite3.OperationalError: + return 0 +``` + +- [ ] **Step 2: Write the failing test `tests/unit/memory/test_schema.py`** + +```python +from __future__ import annotations + +import sqlite3 +import tempfile +from pathlib import Path + +from paperforge.memory.schema import ( + ALL_TABLES, + ensure_schema, + drop_all_tables, + get_schema_version, + CURRENT_SCHEMA_VERSION, +) +from paperforge.memory.db import get_connection + + +def test_ensure_schema_creates_all_tables(): + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + db_path = Path(tmp.name) + try: + conn = get_connection(db_path) + ensure_schema(conn) + cursor = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" + ) + tables = {row["name"] for row in cursor.fetchall()} + for table in ALL_TABLES: + assert table in tables, f"Missing table: {table}" + conn.close() + finally: + db_path.unlink(missing_ok=True) + + +def test_drop_all_tables_clears_all(): + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + db_path = Path(tmp.name) + try: + conn = get_connection(db_path) + ensure_schema(conn) + drop_all_tables(conn) + cursor = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + tables = {row["name"] for row in cursor.fetchall()} + assert tables == set() + conn.close() + finally: + db_path.unlink(missing_ok=True) + + +def test_get_schema_version_returns_zero_when_no_meta(): + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + db_path = Path(tmp.name) + try: + conn = get_connection(db_path) + ensure_schema(conn) + assert get_schema_version(conn) == 0 + conn.close() + finally: + db_path.unlink(missing_ok=True) + + +def test_get_schema_version_returns_stored_value(): + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + db_path = Path(tmp.name) + try: + conn = get_connection(db_path) + ensure_schema(conn) + conn.execute( + "INSERT INTO meta (key, value) VALUES ('schema_version', '1')" + ) + conn.commit() + assert get_schema_version(conn) == 1 + conn.close() + finally: + db_path.unlink(missing_ok=True) + + +def test_schema_version_mismatch_triggers_rebuild_semantics(): + """When stored version != CURRENT, get_schema_version returns a different int.""" + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + db_path = Path(tmp.name) + try: + conn = get_connection(db_path) + ensure_schema(conn) + conn.execute( + "INSERT INTO meta (key, value) VALUES ('schema_version', '99')" + ) + conn.commit() + stored = get_schema_version(conn) + assert stored != CURRENT_SCHEMA_VERSION + conn.close() + finally: + db_path.unlink(missing_ok=True) +``` + +- [ ] **Step 3: Run tests and verify they pass** + +```bash +python -m pytest tests/unit/memory/test_schema.py -v +``` + +- [ ] **Step 4: Commit** + +```bash +git add paperforge/memory/schema.py tests/unit/memory/ +git commit -m "feat(memory): add schema module with table definitions and tests" +``` + +--- + +### Task 4: `paperforge/memory/builder.py` + +**Files:** +- Create: `paperforge/memory/builder.py` +- Create: `tests/unit/memory/test_builder.py` +- Modify: (none) + +- [ ] **Step 1: Write `builder.py`** + +```python +from __future__ import annotations + +import hashlib +import json +import logging +from datetime import datetime, timezone +from pathlib import Path + +from paperforge import __version__ as PF_VERSION +from paperforge.memory.db import get_connection, get_memory_db_path +from paperforge.memory.schema import ( + CURRENT_SCHEMA_VERSION, + ensure_schema, + drop_all_tables, + get_schema_version, +) +from paperforge.worker.asset_index import read_index +from paperforge.worker.asset_state import ( + compute_lifecycle, + compute_maturity, + compute_next_step, +) + +logger = logging.getLogger(__name__) + +PAPER_COLUMNS = [ + "zotero_key", "citation_key", "title", "year", "doi", "pmid", + "journal", "first_author", "authors_json", "abstract", "domain", + "collection_path", "collections_json", + "has_pdf", "do_ocr", "analyze", "ocr_status", "deep_reading_status", + "ocr_job_id", "impact_factor", + "lifecycle", "maturity_level", "maturity_name", "next_step", + "pdf_path", "note_path", "main_note_path", "paper_root", + "fulltext_path", "ocr_md_path", "ocr_json_path", "ai_path", + "deep_reading_md_path", "updated_at", +] + +ASSET_FIELDS = [ + ("pdf", "pdf_path"), + ("formal_note", "note_path"), + ("main_note", "main_note_path"), + ("ocr_fulltext", "fulltext_path"), + ("ocr_meta", "ocr_json_path"), + ("deep_reading", "main_note_path"), + ("ai_dir", "ai_path"), +] + +ALIAS_TYPES = ["zotero_key", "citation_key", "title", "doi"] + + +def compute_hash(items: list[dict]) -> str: + sorted_items = sorted(items, key=lambda e: e["zotero_key"]) + raw = json.dumps(sorted_items, sort_keys=True, ensure_ascii=False) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _resolve_vault_path(vault: Path, rel_path: str) -> Path: + if not rel_path: + return Path() + p = vault / rel_path + return p.resolve() if p.exists() else p + + +def build_from_index(vault: Path) -> dict: + """Read formal-library.json and build/rebuild paperforge.db. + + Returns a dict with counts for reporting. + """ + envelope = read_index(vault) + if envelope is None: + raise FileNotFoundError( + "Canonical index not found. Run paperforge sync --rebuild-index." + ) + # Legacy format: bare list of entries (pre-envelope) + if isinstance(envelope, list): + items = envelope + generated_at = "" + else: + items = envelope.get("items", []) + generated_at = envelope.get("generated_at", "") + if isinstance(items, list) and items and isinstance(items[0], dict): + canonical_hash = compute_hash(items) + else: + canonical_hash = "" + + db_path = get_memory_db_path(vault) + conn = get_connection(db_path, read_only=False) + try: + stored_version = get_schema_version(conn) + if stored_version != CURRENT_SCHEMA_VERSION: + drop_all_tables(conn) + ensure_schema(conn) + + conn.execute("DELETE FROM paper_aliases;") + conn.execute("DELETE FROM paper_assets;") + conn.execute("DELETE FROM papers;") + + now_utc = datetime.now(timezone.utc).isoformat() + papers_count = 0 + assets_count = 0 + aliases_count = 0 + + for entry in items: + zotero_key = entry.get("zotero_key", "") + if not zotero_key: + continue + + lifecycle = str(compute_lifecycle(entry)) + maturity = compute_maturity(entry) + next_step = str(compute_next_step(entry)) + + paper_values = {} + for col in PAPER_COLUMNS: + if col == "authors_json": + paper_values[col] = json.dumps( + entry.get("authors", []), ensure_ascii=False + ) + elif col == "collections_json": + paper_values[col] = json.dumps( + entry.get("collections", []), ensure_ascii=False + ) + elif col == "lifecycle": + paper_values[col] = lifecycle + elif col == "maturity_level": + paper_values[col] = maturity.get("level", 1) + elif col == "maturity_name": + paper_values[col] = maturity.get("level_name", "") + elif col == "next_step": + paper_values[col] = next_step + elif col == "updated_at": + paper_values[col] = generated_at + elif col in ("do_ocr", "analyze"): + val = entry.get(col) + paper_values[col] = 1 if val else 0 + elif col == "has_pdf": + paper_values[col] = 1 if entry.get("has_pdf") else 0 + else: + paper_values[col] = entry.get(col, "") + + placeholders = ", ".join([f":{c}" for c in PAPER_COLUMNS]) + cols = ", ".join(PAPER_COLUMNS) + conn.execute( + f"INSERT OR REPLACE INTO papers ({cols}) VALUES ({placeholders})", + paper_values, + ) + papers_count += 1 + + for asset_type, entry_field in ASSET_FIELDS: + path_val = entry.get(entry_field, "") + if not path_val: + continue + rel_path = str(path_val).replace("\\", "/") + abs_path = _resolve_vault_path(vault, rel_path) + exists = 1 if abs_path.exists() else 0 + + if asset_type == "deep_reading": + if abs_path.exists(): + try: + content = abs_path.read_text(encoding="utf-8") + exists = 1 if "## 🔍 精读" in content else 0 + except Exception: + exists = 0 + + conn.execute( + """INSERT OR REPLACE INTO paper_assets + (paper_id, asset_type, path, exists_on_disk) + VALUES (?, ?, ?, ?)""", + (zotero_key, asset_type, rel_path, exists), + ) + assets_count += 1 + + for alias_type in ALIAS_TYPES: + raw_val = entry.get(alias_type, "") + if not raw_val: + continue + raw_str = str(raw_val) + conn.execute( + """INSERT OR REPLACE INTO paper_aliases + (paper_id, alias, alias_norm, alias_type) + VALUES (?, ?, ?, ?)""", + ( + zotero_key, + raw_str, + raw_str.lower().strip(), + alias_type, + ), + ) + aliases_count += 1 + + meta_upserts = [ + ("schema_version", str(CURRENT_SCHEMA_VERSION)), + ("paperforge_version", PF_VERSION), + ("created_at", now_utc), + ("last_full_build_at", now_utc), + ("canonical_index_hash", canonical_hash), + ("canonical_index_generated_at", generated_at), + ] + for key, value in meta_upserts: + conn.execute( + """INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)""", + (key, value), + ) + + conn.commit() + + return { + "db_path": str(db_path), + "papers_indexed": papers_count, + "assets_indexed": assets_count, + "aliases_indexed": aliases_count, + "schema_version": str(CURRENT_SCHEMA_VERSION), + } + except Exception: + conn.rollback() + raise + finally: + conn.close() +``` + +- [ ] **Step 2: Write the test `tests/unit/memory/test_builder.py`** + +Note: This test needs an actual `formal-library.json` fixture. Use the existing test vault. + +```python +from __future__ import annotations + +import tempfile +from pathlib import Path +from unittest.mock import patch, MagicMock + +from paperforge.memory.builder import build_from_index, compute_hash + + +def test_compute_hash_deterministic(): + items1 = [{"zotero_key": "A"}, {"zotero_key": "B"}] + items2 = [{"zotero_key": "B"}, {"zotero_key": "A"}] + assert compute_hash(items1) == compute_hash(items2) + + +def test_compute_hash_different_for_different_data(): + items1 = [{"zotero_key": "A", "title": "X"}] + items2 = [{"zotero_key": "A", "title": "Y"}] + assert compute_hash(items1) != compute_hash(items2) + + +def test_compute_hash_handles_empty(): + assert compute_hash([]) == compute_hash([]) + assert len(compute_hash([])) == 64 # SHA-256 hex +``` + +- [ ] **Step 3: Run tests** + +```bash +python -m pytest tests/unit/memory/test_builder.py -v +``` + +- [ ] **Step 4: Commit** + +```bash +git add paperforge/memory/builder.py tests/unit/memory/test_builder.py +git commit -m "feat(memory): add builder module that populates SQLite from formal-library.json" +``` + +--- + +### Task 5: `paperforge/memory/query.py` + +**Files:** +- Create: `paperforge/memory/query.py` +- Create: `tests/unit/memory/test_query.py` + +- [ ] **Step 1: Write `query.py`** + +```python +from __future__ import annotations + +import json +import logging +from pathlib import Path + +from paperforge.memory.db import get_connection, get_memory_db_path +from paperforge.memory.schema import get_schema_version, CURRENT_SCHEMA_VERSION +from paperforge.memory.builder import compute_hash +from paperforge.worker.asset_state import compute_health +from paperforge.worker.asset_index import read_index + + +def get_memory_status(vault: Path) -> dict: + """Check paperforge.db health and staleness. + + Returns a dict with: db_exists, schema_ok, fresh, count_match, + paper_count_db, paper_count_index, needs_rebuild. + """ + db_path = get_memory_db_path(vault) + result = { + "db_exists": db_path.exists(), + "schema_ok": False, + "fresh": False, + "count_match": False, + "paper_count_db": 0, + "paper_count_index": 0, + "needs_rebuild": True, + } + if not db_path.exists(): + return result + + conn = get_connection(db_path, read_only=True) + try: + stored_version = get_schema_version(conn) + result["schema_ok"] = stored_version == CURRENT_SCHEMA_VERSION + row = conn.execute("SELECT COUNT(*) as cnt FROM papers").fetchone() + result["paper_count_db"] = row["cnt"] if row else 0 + stored_hash_row = conn.execute( + "SELECT value FROM meta WHERE key = 'canonical_index_hash'" + ).fetchone() + stored_hash = stored_hash_row["value"] if stored_hash_row else "" + except Exception: + return result + finally: + conn.close() + + envelope = read_index(vault) + if envelope is not None: + # Handle legacy format (bare list) + if isinstance(envelope, list): + items = envelope + paper_count = len(items) + index_hash = compute_hash(items) + else: + items = envelope.get("items", []) + paper_count = envelope.get("paper_count", 0) + index_hash = compute_hash(items) + result["paper_count_index"] = paper_count + + # Compare stored hash with computed hash + result["hash_match"] = stored_hash == index_hash + + result["count_match"] = ( + result["paper_count_db"] == result["paper_count_index"] + ) + + result["fresh"] = ( + result["schema_ok"] + and result["count_match"] + and result.get("hash_match", False) + ) + result["needs_rebuild"] = not result["fresh"] + return result + + +def _entry_from_row(row) -> dict: + """Reconstruct an entry dict from a papers row (sqlite3.Row).""" + entry = {k: row[k] for k in row.keys()} + for key in ("has_pdf", "do_ocr", "analyze"): + if key in entry and entry[key] is not None: + entry[key] = bool(entry[key]) + for key in ("authors_json", "collections_json"): + if key in entry and entry[key]: + try: + entry[key[:-5]] = json.loads(entry[key]) + del entry[key] + except json.JSONDecodeError: + logging.warning( + "Corrupted JSON in column %s for paper %s", + key, entry.get("zotero_key", "?"), + ) + return entry + + +def lookup_paper(conn, query: str) -> list[dict]: + """Multi-strategy lookup. Returns list of matching paper dicts.""" + q = query.strip() + results = [] + + for lookup_col in ("zotero_key", "citation_key", "doi"): + row = conn.execute( + f"SELECT * FROM papers WHERE LOWER({lookup_col}) = LOWER(?)", + (q,), + ).fetchone() + if row: + return [_entry_from_row(row)] + + rows = conn.execute( + """SELECT * FROM papers + WHERE LOWER(title) LIKE '%' || LOWER(?) || '%' + LIMIT 20""", + (q,), + ).fetchall() + if rows: + return [_entry_from_row(r) for r in rows] + + rows = conn.execute( + """SELECT p.* FROM papers p + JOIN paper_aliases a ON a.paper_id = p.zotero_key + WHERE a.alias_norm LIKE '%' || LOWER(?) || '%' + LIMIT 20""", + (q,), + ).fetchall() + return [_entry_from_row(r) for r in rows] + + +def get_paper_assets(conn, zotero_key: str) -> list[dict]: + rows = conn.execute( + "SELECT asset_type, path, exists_on_disk FROM paper_assets WHERE paper_id = ?", + (zotero_key,), + ).fetchall() + return [dict(r) for r in rows] + + +def get_paper_status(vault: Path, query: str) -> dict | None: + """Full paper status lookup. Returns dict or None if not found. + + If multiple candidates found, returns a candidate list without full status. + """ + db_path = get_memory_db_path(vault) + if not db_path.exists(): + return None + + conn = get_connection(db_path, read_only=True) + try: + entries = lookup_paper(conn, query) + if not entries: + return None + + # Multiple candidates → return candidate list only (no full status) + if len(entries) > 1: + return { + "resolved": False, + "candidates": [ + { + "zotero_key": e.get("zotero_key"), + "title": e.get("title"), + "year": e.get("year"), + "citation_key": e.get("citation_key"), + "lifecycle": e.get("lifecycle"), + } + for e in entries + ], + } + + entry = entries[0] + assets = get_paper_assets(conn, entry["zotero_key"]) + entry["health"] = compute_health(entry) + entry["assets"] = assets + entry["resolved"] = True + + next_step = entry.get("next_step", "") + zk = entry.get("zotero_key", "") + if next_step == "/pf-deep": + entry["recommended_action"] = f"/pf-deep {zk}" + elif next_step == "ocr": + entry["recommended_action"] = f"paperforge ocr --key {zk}" + elif next_step == "sync": + entry["recommended_action"] = "paperforge sync" + else: + entry["recommended_action"] = None + + return entry + finally: + conn.close() +``` + +- [ ] **Step 2: Write `tests/unit/memory/test_query.py`** + +```python +from __future__ import annotations + +from paperforge.memory.query import get_memory_status + + +def test_get_memory_status_returns_needs_rebuild_when_no_db(): + from pathlib import Path + result = get_memory_status(Path("/nonexistent/vault")) + assert result["db_exists"] is False + assert result["needs_rebuild"] is True +``` + +- [ ] **Step 3: Run tests** + +```bash +python -m pytest tests/unit/memory/test_query.py -v +``` + +- [ ] **Step 4: Commit** + +```bash +git add paperforge/memory/query.py tests/unit/memory/test_query.py +git commit -m "feat(memory): add query module for paper lookup and status check" +``` + +--- + +### Task 6: CLI commands — `memory.py` and `paper_status.py` + +**Files:** +- Create: `paperforge/commands/memory.py` +- Create: `paperforge/commands/paper_status.py` +- Modify: `paperforge/cli.py:258-259` (register parsers) +- Modify: `paperforge/commands/__init__.py:4-13` (register in command dispatch) + +- [ ] **Step 1: Write `paperforge/commands/memory.py`** + +```python +from __future__ import annotations + +import argparse +import sys + +from paperforge.core.errors import ErrorCode +from paperforge.core.result import PFError, PFResult +from paperforge.memory.builder import build_from_index +from paperforge.memory.query import get_memory_status +from paperforge import __version__ as PF_VERSION + + +def run(args: argparse.Namespace) -> int: + vault = args.vault_path + sub_cmd = args.memory_subcommand + + if sub_cmd == "build": + try: + counts = build_from_index(vault) + result = PFResult( + ok=True, + command="memory build", + version=PF_VERSION, + data=counts, + ) + except FileNotFoundError: + result = PFResult( + ok=False, + command="memory build", + version=PF_VERSION, + error=PFError( + code=ErrorCode.PATH_NOT_FOUND, + message="Canonical index not found. Run paperforge sync --rebuild-index.", + ), + next_actions=[ + { + "command": "paperforge sync --rebuild-index", + "reason": "Generate formal-library.json first", + } + ], + ) + except Exception as exc: + result = PFResult( + ok=False, + command="memory build", + version=PF_VERSION, + error=PFError( + code=ErrorCode.INTERNAL_ERROR, + message=str(exc), + ), + ) + if args.json: + print(result.to_json()) + else: + if result.ok: + print(f"Memory built: {result.data}") + else: + print(f"Error: {result.error.message}", file=sys.stderr) + return 0 if result.ok else 1 + + if sub_cmd == "status": + try: + status = get_memory_status(vault) + result = PFResult( + ok=True, + command="memory status", + version=PF_VERSION, + data=status, + ) + except Exception as exc: + result = PFResult( + ok=False, + command="memory status", + version=PF_VERSION, + error=PFError( + code=ErrorCode.INTERNAL_ERROR, + message=str(exc), + ), + ) + if args.json: + print(result.to_json()) + else: + if result.ok: + for k, v in status.items(): + print(f" {k}: {v}") + else: + print(f"Error: {result.error.message}", file=sys.stderr) + return 0 if result.ok else 1 + + print(f"Unknown memory subcommand: {sub_cmd}", file=sys.stderr) + return 1 +``` + +- [ ] **Step 2: Write `paperforge/commands/paper_status.py`** + +```python +from __future__ import annotations + +import argparse +import sys + +from paperforge.core.errors import ErrorCode +from paperforge.core.result import PFError, PFResult +from paperforge.memory.query import get_paper_status +from paperforge import __version__ as PF_VERSION + + +def run(args: argparse.Namespace) -> int: + vault = args.vault_path + query = args.query + + try: + status = get_paper_status(vault, query) + if status is None: + result = PFResult( + ok=False, + command="paper-status", + version=PF_VERSION, + error=PFError( + code=ErrorCode.PATH_NOT_FOUND, + message=f"No paper found for: {query}", + ), + next_actions=[ + { + "command": "paperforge search", + "reason": "Search for papers by keyword", + } + ], + ) + else: + result = PFResult( + ok=True, + command="paper-status", + version=PF_VERSION, + data=status, + ) + except Exception as exc: + result = PFResult( + ok=False, + command="paper-status", + version=PF_VERSION, + error=PFError( + code=ErrorCode.INTERNAL_ERROR, + message=str(exc), + ), + ) + + if args.json: + print(result.to_json()) + else: + if result.ok: + data = result.data + if data.get("resolved"): + print(f"Zotero Key: {data.get('zotero_key', '')}") + print(f"Title: {data.get('title', '')}") + print(f"Year: {data.get('year', '')}") + print(f"Lifecycle: {data.get('lifecycle', '')}") + print(f"Next Step: {data.get('next_step', '')}") + if data.get("candidates"): + print(f"\nMultiple candidates: {len(data['candidates'])}") + for c in data["candidates"]: + print(f" - {c['zotero_key']}: {c['title']} ({c['year']})") + else: + print(f"Error: {result.error.message}", file=sys.stderr) + + return 0 if result.ok else 1 +``` + +- [ ] **Step 3: Register in `cli.py`** + +In `paperforge/cli.py`, at `build_parser()` after line 259 (`p_dash`), add: + +```python + # Memory Layer commands + p_memory = sub.add_parser("memory", help="Manage the Memory Layer") + p_memory_sp = p_memory.add_subparsers(dest="memory_subcommand", required=True) + p_memory_build = p_memory_sp.add_parser("build", help="Build the memory database from canonical index") + p_memory_build.add_argument("--json", action="store_true", help="Output as JSON") + p_memory_status = p_memory_sp.add_parser("status", help="Check memory database status") + p_memory_status.add_argument("--json", action="store_true", help="Output as JSON") + + p_paper_status = sub.add_parser("paper-status", help="Look up a paper's status") + p_paper_status.add_argument("query", help="Paper identifier (zotero_key, DOI, title, alias)") + p_paper_status.add_argument("--json", action="store_true", help="Output as JSON") +``` + +In `main()`, after `if args.command == "dashboard": ...` (around line 468, find the command dispatch section), add: + +```python + if args.command == "memory": + from paperforge.commands.memory import run + return run(args) + + if args.command == "paper-status": + from paperforge.commands.paper_status import run + return run(args) +``` + +(Follow existing dispatch pattern — see how "dashboard" dispatches.) + +- [ ] **Step 4: Register in `commands/__init__.py`** + +In `paperforge/commands/__init__.py`, add to `_COMMAND_REGISTRY`: + +```python + "memory": "paperforge.commands.memory", + "paper-status": "paperforge.commands.paper_status", +``` + +- [ ] **Step 5: Verify CLI registration** + +```bash +paperforge --help +``` +Expected: `memory` and `paper-status` appear in subcommand list. + +```bash +paperforge memory --help +``` +Expected: shows `build` and `status` subcommands. + +```bash +paperforge memory status --help +``` + +- [ ] **Step 6: Commit** + +```bash +git add paperforge/commands/memory.py paperforge/commands/paper_status.py paperforge/cli.py paperforge/commands/__init__.py +git commit -m "feat(cli): add memory build/status and paper-status commands" +``` + +--- + +### Task 7: Integration test + +**Files:** +- Create: `tests/integration/test_memory_workflow.py` + +- [ ] **Step 1: Write integration test** + +```python +from __future__ import annotations + +import pytest +from pathlib import Path + + +@pytest.mark.integration +def test_memory_build_and_status_with_test_vault(test_vault: Path): + """End-to-end: sync → memory build → memory status → paper-status.""" + import subprocess + import json + + pf = ["python", "-m", "paperforge", "--vault", str(test_vault)] + + # 1. Sync to ensure formal-library.json exists + result = subprocess.run(pf + ["sync", "--json"], capture_output=True, text=True) + # If sync fails, skip (test vault may not have exports) + if result.returncode != 0: + pytest.skip("Sync failed — test vault may lack export files") + + # 2. Memory build + result = subprocess.run(pf + ["memory", "build", "--json"], capture_output=True, text=True) + assert result.returncode == 0 + data = json.loads(result.stdout) + assert data["ok"] is True + assert data["data"]["papers_indexed"] > 0 + + # 3. Memory status + result = subprocess.run(pf + ["memory", "status", "--json"], capture_output=True, text=True) + assert result.returncode == 0 + data = json.loads(result.stdout) + assert data["data"]["fresh"] is True + assert data["data"]["needs_rebuild"] is False +``` + +- [ ] **Step 2: Run integration test** (requires test vault) + +```bash +python -m pytest tests/integration/test_memory_workflow.py -v -m integration +``` + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/test_memory_workflow.py +git commit -m "test(memory): add integration test for memory build/status workflow" +``` + +--- + +### Task 8: Final verification — run full test suite + +- [ ] **Step 1: Run all tests** + +```bash +python -m pytest tests/unit/ tests/integration/ -q --tb=short +``` + +Expected: All tests pass, no regressions. + +- [ ] **Step 2: Run ruff lint** + +```bash +ruff check paperforge/memory/ paperforge/commands/memory.py paperforge/commands/paper_status.py --fix && ruff format paperforge/memory/ paperforge/commands/memory.py paperforge/commands/paper_status.py +``` + +- [ ] **Step 3: Manual smoke test with real vault** + +```bash +paperforge memory build --json +paperforge memory status --json +paperforge paper-status "aaronStimulationGrowthFactor2004" --json +``` + +Expected: Real data flows through, paper status shows lifecycle, next_step, assets. + +--- + +## Summary + +| Task | Files Created | Files Modified | Tests | +|------|--------------|----------------|-------| +| 1. Config path | — | `config.py` | manual | +| 2. db.py | `memory/__init__.py`, `memory/db.py` | — | manual | +| 3. schema.py | `memory/schema.py` | — | `test_schema.py` (4 tests) | +| 4. builder.py | `memory/builder.py` | — | `test_builder.py` (3 tests) | +| 5. query.py | `memory/query.py` | — | `test_query.py` (1 test) | +| 6. CLI | `commands/memory.py`, `commands/paper_status.py` | `cli.py`, `commands/__init__.py` | — | +| 7. Integration | `tests/integration/test_memory_workflow.py` | — | 1 test | +| 8. Verification | — | — | full suite + lint | diff --git a/docs/superpowers/specs/2026-05-12-memory-layer-design.md b/docs/superpowers/specs/2026-05-12-memory-layer-design.md new file mode 100644 index 00000000..afb80fc1 --- /dev/null +++ b/docs/superpowers/specs/2026-05-12-memory-layer-design.md @@ -0,0 +1,279 @@ +# Memory Layer — Design Spec + +> **Status:** Approved | **Date:** 2026-05-12 +> **Review:** Passed (v2 — 5 BLOCKER, 3 MAJOR, 6 MINOR resolved) + +## Goal + +Add a SQLite-backed Memory Layer to PaperForge as a derived, rebuildable global index that serves +dashboard, resolver, agent-context, and search commands. + +## Architecture + +``` +Zotero/BetterBibTeX → exports/*.json + ↓ +formal-library.json (Canonical Index — source of truth, already exists) + ↓ +paperforge.db (Memory Layer — derived, rebuildable SQLite index) + ↓ +paper-status / dashboard / agent-context / search / retrieve +``` + +**Core principle:** `paperforge.db` is a derived index, not the source of truth. +It can be safely deleted and rebuilt from `formal-library.json` at any time. + +## Phase 1 Scope + +**Tables:** `meta`, `papers`, `paper_assets`, `paper_aliases` + +**Commands:** +- `paperforge memory build --json` +- `paperforge memory status --json` +- `paperforge paper-status --json` + +**NOT in Phase 1:** FTS5, chunk retrieval, embedding, `paperforge.db → Markdown` writes, +agent-context, dashboard integration. + +## SQLite Location + +``` +/PaperForge/indexes/paperforge.db +``` +(same directory as `formal-library.json`) + +Register a new path key `"memory_db"` in `config.py:paperforge_paths()` pointing to +`paperforge / "indexes" / "paperforge.db"`. Do not reuse the existing `"index"` key. + +## Schema + +### Connection settings + +- `PRAGMA journal_mode=WAL;` — allow concurrent reads during rebuild +- `PRAGMA foreign_keys=ON;` + +### meta + +```sql +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +``` + +Stores: `schema_version` (integer), `paperforge_version`, `created_at`, `last_full_build_at`, +`canonical_index_hash`, `canonical_index_generated_at`. + +### Schema versioning strategy + +On `paperforge memory build`, if the stored `schema_version` in `meta` does not match the +current version, DROP all tables and rebuild from scratch. `paperforge.db` is a derived index +— full rebuild is always safe. This mirrors `formal-library.json`'s schema-version-check +pattern in `asset_index.py:475-480`. + +Initial schema version: `1`. + +### papers + +One row per paper. Columns directly map to `_build_entry()` entry dict fields. +`asset_state.py` pure functions (`compute_lifecycle`, `compute_health`, `compute_maturity`, +`compute_next_step`) are called at **build time** on each entry dict to populate derived columns. + +```sql +CREATE TABLE IF NOT EXISTS papers ( + zotero_key TEXT PRIMARY KEY, + citation_key TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL, + year TEXT, + doi TEXT, + pmid TEXT, + journal TEXT, + first_author TEXT, + authors_json TEXT, -- json.dumps(entry["authors"], ensure_ascii=False) + abstract TEXT, + domain TEXT, + collection_path TEXT, + collections_json TEXT, -- json.dumps(entry["collections"], ensure_ascii=False) + has_pdf INTEGER NOT NULL DEFAULT 0, + do_ocr INTEGER, + analyze INTEGER, + ocr_status TEXT, + deep_reading_status TEXT, + ocr_job_id TEXT, + impact_factor REAL, + lifecycle TEXT, -- compute_lifecycle(entry) → "indexed"|"pdf_ready"|"fulltext_ready"|"deep_read_done" + maturity_level INTEGER, -- compute_maturity(entry)["level"] → 1-4 + maturity_name TEXT, -- compute_maturity(entry)["level_name"] + next_step TEXT, -- compute_next_step(entry) → "sync"|"ocr"|"/pf-deep"|"ready" + pdf_path TEXT, + note_path TEXT, + main_note_path TEXT, + paper_root TEXT, + fulltext_path TEXT, + ocr_md_path TEXT, + ocr_json_path TEXT, + ai_path TEXT, + deep_reading_md_path TEXT, + updated_at TEXT -- envelope["generated_at"] from formal-library.json +); +``` + +Indexes: +```sql +CREATE INDEX IF NOT EXISTS idx_papers_zotero_key ON papers(zotero_key); +CREATE INDEX IF NOT EXISTS idx_papers_citation_key ON papers(citation_key); +CREATE INDEX IF NOT EXISTS idx_papers_doi ON papers(doi); +CREATE INDEX IF NOT EXISTS idx_papers_domain ON papers(domain); +CREATE INDEX IF NOT EXISTS idx_papers_year ON papers(year); +CREATE INDEX IF NOT EXISTS idx_papers_ocr_status ON papers(ocr_status); +CREATE INDEX IF NOT EXISTS idx_papers_deep_status ON papers(deep_reading_status); +CREATE INDEX IF NOT EXISTS idx_papers_lifecycle ON papers(lifecycle); +CREATE INDEX IF NOT EXISTS idx_papers_next_step ON papers(next_step); +``` + +**Important notes about column→entry mapping:** + +- `maturity_level` = `compute_maturity(entry)["level"]` (scalar 1-4, not the full dict) +- `updated_at` = the envelope's `generated_at` timestamp from `formal-library.json` (shared across all papers in a build) +- `lifecycle` values: `"indexed"`, `"pdf_ready"`, `"fulltext_ready"`, `"deep_read_done"` — these are NOT all members of the `Lifecycle` enum in `core/state.py` (which has `OCR_READY`, `ANALYZE_READY`, `ERROR_STATE` that are never produced). Use plain string comparison, not enum membership. +- `ai_context_ready` is a pre-seeded zero in `summarize_index()` (`asset_index.py:644`) but is never produced by `compute_lifecycle()`. Keep the zero bucket for Phase 3 compatibility but document it as reserved. + +**Health dimensions** (`pdf_health`, `ocr_health`, `note_health`, `asset_health`) are NOT stored in the papers table. They are computed at query time via `asset_state.compute_health(entry_dict)`. The `paper-status` command reconstructs the entry dict from SQLite columns, then calls `compute_health()` in-process. + +### paper_assets + +```sql +CREATE TABLE IF NOT EXISTS paper_assets ( + paper_id TEXT NOT NULL, + asset_type TEXT NOT NULL, + path TEXT NOT NULL, + exists_on_disk INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (paper_id, asset_type), + FOREIGN KEY (paper_id) REFERENCES papers(zotero_key) +); +``` + +Asset types and their source fields: + +| asset_type | source in entry dict | notes | +| -------------- | -------------------------- | --------------------------------------------------------- | +| `pdf` | `pdf_path` | wiki-link; check existence via filesystem | +| `formal_note` | `note_path` | relative vault path | +| `main_note` | `main_note_path` | workspace `{key}.md` | +| `ocr_fulltext` | `fulltext_path` | copied from `ocr/{key}/fulltext.md` | +| `ocr_meta` | derived from `ocr_json_path` | `ocr/{key}/meta.json` | +| `deep_reading` | `main_note_path` | checks for `## 🔍 精读` section within main note (NOT a separate file; `deep_reading_path` is deprecated and always empty) | +| `ai_dir` | `ai_path` | workspace `ai/` directory | + +### paper_aliases + +```sql +CREATE TABLE IF NOT EXISTS paper_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + paper_id TEXT NOT NULL, + alias TEXT NOT NULL, + alias_norm TEXT NOT NULL, + alias_type TEXT NOT NULL, + FOREIGN KEY (paper_id) REFERENCES papers(zotero_key) +); +``` + +Alias types (Phase 1): + +| alias_type | source | normalized to lowercase | +| ------------- | ----------------- | --------------------------- | +| `zotero_key` | `entry["zotero_key"]` | as-is (uppercase) | +| `citation_key` | `entry["citation_key"]` | as-is (case-sensitive) | +| `title` | `entry["title"]` | `.lower().strip()` | +| `doi` | `entry["doi"]` | `.lower().strip()` | + +## Commands + +### `paperforge memory build --json` + +1. Resolve vault path +2. Read `formal-library.json` (canonical index envelope) via `read_index(vault)` +3. If index is `None` or missing → return `PFResult(ok=False, error=PFError(code=PATH_NOT_FOUND, message="Canonical index not found. Run paperforge sync --rebuild-index."))` +4. Extract `items` list and envelope metadata +5. Create/open `paperforge.db` (WAL mode) +6. If stored `schema_version` != current → DROP all tables +7. Create tables if not exist +8. Upsert `meta` rows: `schema_version`, `paperforge_version`, `created_at`, `last_full_build_at`, `canonical_index_generated_at` +9. Compute `canonical_index_hash` = SHA-256 of `json.dumps(sorted(items, key=lambda e: e["zotero_key"]), sort_keys=True, ensure_ascii=False)`; store in `meta` +10. For each entry in `items`: + - Insert/upsert into `papers` + - Insert/upsert into `paper_assets` (check `exists_on_disk` via `Path.exists()`) + - Insert/upsert into `paper_aliases` +11. Return `PFResult(ok=True, data={...})` with `papers_indexed`, `assets_indexed`, `aliases_indexed` counts + +**PFResult.next_actions format** (must match `core/result.py:26` — `list[dict]`): +```json +{ + "next_actions": [ + {"command": "paperforge paper-status --json", "reason": "Look up a specific paper"} + ] +} +``` + +### `paperforge memory status --json` + +Check: +- `paperforge.db` exists → `db_exists: bool` +- `schema_version` matches current → `schema_ok: bool` +- `canonical_index_hash` matches computed hash of current `formal-library.json` → `fresh: bool` +- Paper count matches `envelope["paper_count"]` → `count_match: bool` +- Any check fails → `needs_rebuild: true` + +Return `PFResult(ok=True, data={...})`. + +### `paperforge paper-status --json` + +**Resolution is short-circuit:** stop at the first step that returns ≥1 result. + +Resolution order: +1. Exact match on `zotero_key` (case-insensitive) +2. Exact match on `citation_key` (case-insensitive) +3. Exact match on `doi` (case-insensitive) +4. LIKE match on `title_norm` or normalized alias (`%%`) +5. Fallback: search `paper_aliases.alias_norm` + +Behavior by result count: +- **0 results:** `PFResult(ok=False, error=NOT_FOUND, next_actions=[{"command": "paperforge search", ...}])` +- **1 result:** Full status with paper metadata, assets, lifecycle, next_step, recommended action +- **>1 results:** Candidate list only (no full status details) + +Full status response includes: +- Paper metadata (title, year, authors, doi, journal, domain, abstract) +- Asset status (exists_on_disk for each asset type) +- Lifecycle state +- Health dimensions (computed at query time via `compute_health()`) +- Maturity level and name +- `next_step` with recommended action +- `recommended_action`: e.g., `"/pf-deep ABCDEFG"` or `"paperforge sync"` or `"paperforge ocr"` + +## Integration Points + +### With sync +After `paperforge sync` completes, optionally refresh memory for changed keys. +Not automatic in Phase 1. + +### With dashboard +Dashboard should prefer `paperforge.db` for stats, fallback to file scanning. +This integration is deferred to Phase 2. + +### With agent +Agent skill bootstrap runs `paperforge agent-context --compact --json` first. +This command is deferred to Phase 3. + +## Constraints + +1. `paperforge.db` is a derived index — deletable, rebuildable +2. No SQLite → Markdown writes in Phase 1 +3. Reuse `asset_state.py` pure functions (compute_lifecycle, compute_health, compute_maturity, compute_next_step) +4. Health dimensions are computed at query time via `compute_health()`, not stored in SQLite +5. All `--json` output uses PFResult envelope (respecting `next_actions: list[dict]` contract) +6. SQLite connection uses WAL mode for concurrent reads +7. No external database services — only Python stdlib `sqlite3` +8. No PDF/image binary storage +9. No embedding or vector DB +10. Schema version mismatch → full drop-and-rebuild (derived index, always safe) diff --git a/manifest.json b/manifest.json index b535c329..af16719a 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "paperforge", "name": "PaperForge", - "version": "1.5.5", + "version": "1.5.6rc1", "minAppVersion": "1.9.0", "description": "PaperForge — Zotero literature pipeline. Sync PDFs, run OCR, and read with AI-assisted deep reading.", "author": "Lin Zhaoxuan", diff --git a/paperforge/__init__.py b/paperforge/__init__.py index 8f670b8f..85369c30 100644 --- a/paperforge/__init__.py +++ b/paperforge/__init__.py @@ -1,3 +1,3 @@ """paperforge — PaperForge package.""" -__version__ = "1.5.5" +__version__ = "1.5.6rc1" diff --git a/paperforge/plugin/manifest.json b/paperforge/plugin/manifest.json index b535c329..af16719a 100644 --- a/paperforge/plugin/manifest.json +++ b/paperforge/plugin/manifest.json @@ -1,7 +1,7 @@ { "id": "paperforge", "name": "PaperForge", - "version": "1.5.5", + "version": "1.5.6rc1", "minAppVersion": "1.9.0", "description": "PaperForge — Zotero literature pipeline. Sync PDFs, run OCR, and read with AI-assisted deep reading.", "author": "Lin Zhaoxuan",