bump: 1.5.5 -> 1.5.6rc1 (Memory Layer RC1)

This commit is contained in:
Research Assistant 2026-05-12 18:11:37 +08:00
parent 135eaaa084
commit 8335bce992
10 changed files with 2188 additions and 3 deletions

37
.github/workflows/ci-chaos.yml vendored Normal file
View file

@ -0,0 +1,37 @@
name: Chaos Tests (L6)
on:
schedule:
# Weekly: Sunday 06:00 UTC
- cron: "0 6 * * 0"
workflow_dispatch:
# Manual trigger from GitHub UI
jobs:
chaos-tests:
name: Chaos / Destructive Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install package with test dependencies
run: |
pip install -e ".[test]"
pip install pytest pytest-timeout responses PyYAML
- name: Run chaos tests
run: |
python -m pytest tests/chaos/ -m chaos -v --tb=long --timeout=120 \
--junit-xml=chaos-results.xml
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: chaos-test-results
path: chaos-results.xml

114
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,114 @@
# CI — Simplified Pipeline
# Runs on push/PR to master. All tests run (no -x early exit).
name: CI
on:
push:
branches: [main, master]
paths-ignore:
- "**.md"
- "docs/**"
pull_request:
branches: [main, master]
env:
PYTHONIOENCODING: utf-8
jobs:
# ---------------------------------------------------------------------------
# L0 — Version consistency check
# ---------------------------------------------------------------------------
version-check:
name: L0 — Version Sync
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install package
run: pip install -e .
- name: Check version consistency
run: python scripts/check_version_sync.py
# ---------------------------------------------------------------------------
# L1 — Unit tests (3 OS x 1 Python)
# ---------------------------------------------------------------------------
unit-tests:
name: L1 — Unit Tests (${{ matrix.os }}, py3.11)
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install package with test deps
run: pip install -e ".[test]"
- name: Run unit tests
shell: bash
run: |
python -m pytest tests/ \
--ignore=tests/sandbox \
--ignore=tests/cli \
--ignore=tests/e2e \
--ignore=tests/journey \
--ignore=tests/chaos \
--ignore=tests/audit \
-v --tb=short --timeout=60
# ---------------------------------------------------------------------------
# L3 — Plugin tests (Vitest)
# ---------------------------------------------------------------------------
plugin-tests:
name: L3 — Plugin Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: "npm"
cache-dependency-path: paperforge/plugin/package-lock.json
- run: npm ci
working-directory: paperforge/plugin
- run: npx vitest run --reporter=verbose
working-directory: paperforge/plugin
# ---------------------------------------------------------------------------
# L4 — E2E + Audit
# ---------------------------------------------------------------------------
e2e-tests:
name: L4 — E2E + Audit
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install package with test deps
run: pip install -e ".[test]"
- name: Run E2E tests
run: python -m pytest tests/e2e/ -m e2e -v --tb=short --timeout=120
- name: Run audit tests
run: python -m pytest tests/audit/ -m audit -v --tb=short --timeout=120
# ---------------------------------------------------------------------------
# Merge gate
# ---------------------------------------------------------------------------
alls-green:
name: All Checks Passed
if: always()
needs:
- unit-tests
- plugin-tests
runs-on: ubuntu-latest
steps:
- uses: re-actors/alls-green@v1.2.2
with:
allowed-skips: version-check
jobs: ${{ toJSON(needs) }}

42
.github/workflows/release.yml vendored Normal file
View file

@ -0,0 +1,42 @@
# Auto-release Obsidian plugin on tag push
# Triggered by tags like v1.4.18. Runs plugin tests, then creates a GitHub Release
# with the 4 required Obsidian plugin files.
name: Release
on:
push:
tags:
- "v*"
jobs:
release:
name: Release Plugin
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
cache: "npm"
cache-dependency-path: paperforge/plugin/package-lock.json
- name: Run plugin tests
working-directory: paperforge/plugin
run: |
npm ci
npx vitest run --reporter=verbose
- name: Create Release
uses: softprops/action-gh-release@v2
with:
name: ${{ github.ref_name }}
generate_release_notes: true
files: |
paperforge/plugin/main.js
paperforge/plugin/styles.css
paperforge/plugin/manifest.json
paperforge/plugin/versions.json

View file

@ -0,0 +1,101 @@
---
phase: memory-layer-plan-v3-quick-check
reviewed: 2026-05-12T09:25:08Z
depth: standard
files_reviewed: 1
files_reviewed_list:
- docs/superpowers/plans/2026-05-12-memory-layer.md
findings:
critical: 0
warning: 1
info: 0
total: 1
status: issues_found
---
# Phase: Memory Layer Plan v3 Quick Check
**Reviewed:** 2026-05-12T09:25:08Z
**Depth:** standard (plan-only, cross-referenced against codebase for `--key` validation)
**Files Reviewed:** 1
**Status:** ISSUES_FOUND (1 WARNING remaining)
---
## Summary
Quick final check of the implementation plan after v3 review fixes. All 5 named issues from the prior review are **confirmed fixed**. The plan incorporates all 14 fixes from the original v1 deep review (5 CR + 5 WR + 4 IN). One new WARNING-level issue identified in the `_entry_from_row` function.
---
## Named Issue Verification
| Issue | Status | Evidence |
|-------|--------|----------|
| **N-BLKR-01**: hash query inside try block | **FIXED** | Lines 713-716 — `stored_hash_row = conn.execute(...)` is inside the `try:` block at line 708 |
| **N-BLKR-02**: NameError on `status` in memory.py | **FIXED** | Line 985 — `if result.ok:` guards access to `status` on line 986. `result` is always assigned in both try/except branches |
| **N-WRN-01**: paper_status empty fields for unresolved | **FIXED** | Line 1055 — `if data.get("resolved"):` guards detailed field printing |
| **N-INFO-01**: private `_compute_hash` renamed to `compute_hash` | **FIXED** | Line 451 — `def compute_hash(...)` (public). Line 683 — `from paperforge.memory.builder import compute_hash` |
| **N-INFO-02**: JSON decode logged with `logging.warning` | **FIXED** | Lines 762-764 — `logging.warning("Corrupted JSON in column %s for paper %s", key, ...)` |
All 5 named issues from the prior review are resolved in the plan.
---
## Original v1 Review Issue Verification (bonus)
Cross-checked all 14 issues from `2026-05-12-memory-layer-REVIEW.md`:
| Issue | Status |
|-------|--------|
| CR-01: `make_result` import | **FIXED** — line 910 imports only `PFError, PFResult` |
| CR-02: hash not checked | **FIXED** — lines 713-746 compare stored hash vs computed |
| CR-03: legacy format crash in builder | **FIXED** — lines 475-480 handle `isinstance(envelope, list)` |
| CR-04: legacy format crash in query | **FIXED** — lines 725-733 handle bare list |
| CR-05: Windows-path URI bug | **FIXED** — line 122 uses `db_path.as_posix()` |
| WR-01: `--force` flag | **FIXED** — removed from CLI parser (lines 1080-1082) |
| WR-02: ambiguous query returns full status | **FIXED** — lines 823-838 return candidates only when >1 |
| WR-03: recommended_action missing | **FIXED** — lines 846-855 compute concrete action strings |
| WR-04: zero test coverage | **REMAINS** — plan still has only 4 schema + 3 hash tests |
| WR-05: CLI dispatch pattern | **FIXED** — lines 1089-1099 use simple dispatch |
| IN-01: unused compute_health import | **FIXED** — removed from builder imports (lines 417-422) |
| IN-02: _COMMAND_REGISTRY not consumed | **REMAINS** — still present but rate-limited to INFO |
| IN-03: compute_hash .get vs direct | **FIXED** — line 452 uses `e["zotero_key"]` (direct access) |
| IN-04: fragile rstrip("_json") | **FIXED** — line 760 uses `key[:-5]` instead of `rstrip` |
---
## Warnings
### WR-V3-01: Data silently lost when JSON decode fails in `_entry_from_row`
**File:** `docs/superpowers/plans/2026-05-12-memory-layer.md:759-760`
**Issue:** When `json.loads()` raises `JSONDecodeError`, `entry.pop(key)` has already executed — the original `_json` column value is removed from the result dict and never restored. The field disappears silently from query output.
```python
# Current (plan line 759-760)
try:
entry[key[:-5]] = json.loads(entry.pop(key)) # pop() happens BEFORE json.loads()
except json.JSONDecodeError:
logging.warning(...) # original value already lost
```
**Fix:**
```python
# Pop first, then try to decode, restore on failure
raw = entry.pop(key)
try:
entry[key[:-5]] = json.loads(raw)
except json.JSONDecodeError:
entry[key] = raw # keep original JSON string visible
logging.warning(
"Corrupted JSON in column %s for paper %s",
key, entry.get("zotero_key", "?"),
)
```
---
_Reviewed: 2026-05-12T09:25:08Z_
_Reviewer: VT-OS/OPENCODE (gsd-code-reviewer)_
_Depth: standard_

View file

@ -0,0 +1,377 @@
---
phase: memory-layer-plan-review
reviewed: 2026-05-12T18:30:00Z
depth: deep
files_reviewed: 9
files_reviewed_list:
- docs/superpowers/plans/2026-05-12-memory-layer.md
- docs/superpowers/specs/2026-05-12-memory-layer-design.md
- paperforge/config.py
- paperforge/cli.py
- paperforge/commands/__init__.py
- paperforge/core/result.py
- paperforge/core/errors.py
- paperforge/worker/asset_state.py
- paperforge/worker/asset_index.py
findings:
critical: 5
warning: 5
info: 4
total: 14
status: issues_found
---
# Phase: Memory Layer Plan Review
**Reviewed:** 2026-05-12T18:30:00Z
**Depth:** deep (cross-file analysis with import graph tracing)
**Files Reviewed:** 9
**Status:** ISSUES_FOUND
## Verdict: ISSUES_FOUND
5 BLOCKER, 5 WARNING, 4 INFO issues detected. Plan must not be executed until BLOCKER items are resolved.
---
## Summary
The plan maps spec requirements to tasks with reasonable granularity, and the overall architecture (SQLite under `paperforge/memory/`, derived from `formal-library.json`, PFResult-enveloped CLI) is sound. However, the cross-file trace against the actual codebase reveals **five BLOCKER defects** — a non-existent import, a missing spec-critical hash check, two crash-on-legacy-format scenarios, and a Windows-path URI bug. Five WARNING-level issues include an unimplemented `--force` flag, a behavioral divergence from spec for ambiguous queries, a missing `recommended_action` field, near-zero test coverage for business logic, and an inconsistent CLI dispatch pattern.
---
## Critical Issues
### CR-01: Import of non-existent `make_result` in `memory.py`
**File:** Plan Task 6, Step 1 (`paperforge/commands/memory.py` line 4)
**Issue:** The plan code imports `make_result` from `paperforge.core.result`:
```python
from paperforge.core.result import PFError, PFResult, make_result
```
`make_result` is **not defined anywhere** in the codebase. Verified by grep of the entire `paperforge/` tree — zero matches. `core/result.py` (lines 1-79) exports only `PFError` and `PFResult`. This would cause `ImportError` at runtime on every invocation of `paperforge memory`.
**Fix:**
```python
# Remove make_result from the import line — it's never used in the function body either.
from paperforge.core.result import PFError, PFResult
```
---
### CR-02: `get_memory_status` does not check `canonical_index_hash`
**File:** Plan Task 5, Step 1 (`paperforge/memory/query.py`, `get_memory_status()`)
**Issue:** The spec (Design Spec lines 221-226) explicitly requires `memory status` to verify `canonical_index_hash` against the SHA-256 of the current `formal-library.json`:
> - `canonical_index_hash` matches computed hash of current `formal-library.json``fresh: bool`
The plan's implementation (lines 678-717) computes `fresh` as only:
```python
result["fresh"] = result["schema_ok"] and result["count_match"]
```
The `canonical_index_hash` stored in `meta` during build is never read back and never compared. The status command will report `fresh: true` even when the canonical index has changed since the last build — giving a falsely green "fresh" signal that causes stale paper-status results.
**Fix:** In `get_memory_status()` after the read-only connection is opened, add:
```python
# Read stored hash from meta
stored_hash_row = conn.execute(
"SELECT value FROM meta WHERE key = 'canonical_index_hash'"
).fetchone()
stored_hash = stored_hash_row["value"] if stored_hash_row else ""
# Recompute hash from current index
envelope = read_index(vault)
items = envelope.get("items", []) if isinstance(envelope, dict) else []
from paperforge.memory.builder import _compute_hash
current_hash = _compute_hash(items) if items else ""
result["hash_match"] = stored_hash == current_hash
result["fresh"] = result["schema_ok"] and result["count_match"] and result["hash_match"]
```
---
### CR-03: `build_from_index` crashes on legacy-format (bare list) index
**File:** Plan Task 4, Step 1 (`paperforge/memory/builder.py`, line 467-471)
**Issue:** `read_index(vault)` in `asset_index.py` (line 160-176) can return a **bare list** (legacy pre-v1.6 format). The `build_from_index` function only checks for `None`:
```python
envelope = read_index(vault)
if envelope is None:
raise FileNotFoundError(...)
items = envelope.get("items", []) # <-- CRASH: list has no .get()
```
If the vault has a legacy-format `formal-library.json` (not yet migrated by a sync run), `envelope` is a `list`, and `envelope.get(...)` raises `AttributeError`. The existing codebase has `is_legacy_format()` and `migrate_legacy_index()` in `asset_index.py` (lines 178-212) specifically for this case.
**Fix:** Add legacy format detection after the `None` check:
```python
envelope = read_index(vault)
if envelope is None:
raise FileNotFoundError(
"Canonical index not found. Run paperforge sync --rebuild-index."
)
from paperforge.worker.asset_index import is_legacy_format
if is_legacy_format(envelope):
raise FileNotFoundError(
"Canonical index is in legacy (bare-list) format. "
"Run paperforge sync --rebuild-index to migrate."
)
items = envelope.get("items", [])
generated_at = envelope.get("generated_at", "")
```
---
### CR-04: `get_memory_status` crashes on legacy-format index
**File:** Plan Task 5, Step 1 (`paperforge/memory/query.py`, line 708-713)
**Issue:** Same legacy-format crash as CR-03, but in the read path:
```python
envelope = read_index(vault)
if envelope:
result["paper_count_index"] = envelope.get("paper_count", 0) # CRASH on list
```
A bare-list envelope causes `AttributeError`.
**Fix:** Add the same `is_legacy_format` guard:
```python
envelope = read_index(vault)
if envelope and isinstance(envelope, dict):
result["paper_count_index"] = envelope.get("paper_count", 0)
...
```
---
### CR-05: Windows-path URI incompatibility in `get_connection` read-only mode
**File:** Plan Task 2, Step 2 (`paperforge/memory/db.py`, line 122-123)
**Issue:**
```python
uri = f"file:{db_path}?mode=ro" if read_only else str(db_path)
conn = sqlite3.connect(uri, uri=read_only)
```
On Windows, `db_path` contains backslashes (e.g., `D:\Vault\System\PaperForge\indexes\paperforge.db`). The constructed URI `file:D:\Vault\...?mode=ro` is NOT a valid [RFC 8089 file URI](https://datatracker.ietf.org/doc/html/rfc8089). SQLite's URI parser requires either `file:///D:/...` (authority path) or `file:D:/...` (local path with forward slashes). With backslashes, `sqlite3.connect(..., uri=True)` may fail with `sqlite3.OperationalError: unable to open database file` or silently misinterpret the path.
**Fix:** Normalize the path to use forward slashes before constructing the URI:
```python
def get_connection(db_path: Path, read_only: bool = False) -> sqlite3.Connection:
if read_only:
# Windows-safe: convert to forward slashes for SQLite URI parser
posix_path = str(db_path.resolve()).replace("\\", "/")
uri = f"file:{posix_path}?mode=ro"
else:
uri = str(db_path)
conn = sqlite3.connect(uri, uri=read_only)
conn.row_factory = sqlite3.Row
if not read_only:
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA foreign_keys=ON;")
return conn
```
---
## Warnings
### WR-01: `--force` flag on `memory build` defined but never implemented
**File:** Plan Task 6, Step 3 (cli.py parser) + Task 4 builder
**Issue:** The CLI parser adds `--force` to `memory build`:
```python
p_memory_build.add_argument("--force", action="store_true", help="Force rebuild")
```
Neither `memory.run()` nor `build_from_index()` checks `args.force`. The builder always deletes all paper data and rebuilds (lines 486-488), making `--force` redundant for the current logic. However, a future optimization that caches unchanged entries would make `--force` meaningful. Either implement the flag or remove it — dead CLI interfaces degrade user experience and create maintenance debt.
**Fix:** Either (a) remove the `--force` argument entirely from the parser, or (b) wire it through:
```python
# In builder.py: add force parameter
def build_from_index(vault: Path, force: bool = False) -> dict:
...
if force:
drop_all_tables(conn)
...
# In memory.py:
counts = build_from_index(vault, force=getattr(args, "force", False))
```
---
### WR-02: Ambiguous query (>1 results) returns full status instead of candidate list only
**File:** Plan Task 5 (`paperforge/memory/query.py`, `get_paper_status()`, lines 775-793)
**Issue:** The spec (Design Spec line 243) states:
> **>1 results:** Candidate list only (no full status details)
The plan returns full status for the first match PLUS the candidate list:
```python
entry = entries[0]
assets = get_paper_assets(conn, entry["zotero_key"])
entry["health"] = compute_health(entry) # Full status details computed
entry["candidates"] = entries if len(entries) > 1 else None
entry["assets"] = assets
return entry
```
And the CLI output (paper_status.py lines 986-991) always prints title/year/lifecycle/next_step — even when multiple candidates exist. This violates the spec's "candidate list only" requirement for ambiguous queries.
**Fix:** When `len(entries) > 1`, return candidate summary only:
```python
if len(entries) > 1:
return {
"candidates": [
{
"zotero_key": e.get("zotero_key", ""),
"title": e.get("title", ""),
"year": e.get("year", ""),
"doi": e.get("doi", ""),
"domain": e.get("domain", ""),
}
for e in entries
],
"candidate_count": len(entries),
}
```
---
### WR-03: `recommended_action` field missing from paper-status output
**File:** Plan Task 5 (`paperforge/memory/query.py`, `get_paper_status()`) + Task 6 (`paper_status.py`)
**Issue:** The spec (Design Spec lines 252-253) requires:
> `recommended_action`: e.g., `"/pf-deep ABCDEFG"` or `"paperforge sync"` or `"paperforge ocr"`
The plan only returns `entry["next_step"]` (e.g., `"/pf-deep"`) in the output but never computes a concrete `recommended_action` like `"/pf-deep ABCDEFG"`. The spec implies this should be a ready-to-use command string with the paper key substituted in.
**Fix:** In `get_paper_status()`, after computing health, add:
```python
step = entry.get("next_step", "")
zkey = entry.get("zotero_key", "")
action_map = {
"/pf-deep": f"/pf-deep {zkey}",
"ocr": f"paperforge ocr --key {zkey}",
"sync": "paperforge sync",
"repair": "paperforge repair",
"ready": "Ready — no action needed",
}
entry["recommended_action"] = action_map.get(step, step)
```
---
### WR-04: Core business logic functions have zero test coverage
**File:** Plan Tasks 4-5 (test files)
**Issue:** The plan specifies 8 tests total:
- 4 schema tests (table creation/deletion/schema version) — good
- 3 builder tests — but ALL three test only `_compute_hash`, a 10-line helper. `build_from_index()` (~150 lines) has **zero tests**.
- 1 query test — only tests `get_memory_status()` with a nonexistent vault path. `lookup_paper()`, `get_paper_assets()`, `get_paper_status()`, and `_entry_from_row()` have **zero tests**.
Untested edge cases include: empty items list, schema version mismatch trigger, corrupt JSON in authors/collections, exact zotero_key lookup, DOI lookup, title substring search, no-results path, asset reconstruction with None values.
**Fix:** Add at minimum:
- `test_build_from_index_empty_items()` — ensure handles empty index gracefully
- `test_build_from_index_schema_mismatch()` — verify drop+rebuild on version change
- `test_build_from_index_populates_correctly()` — build from a mock envelope, verify paper count/asset count
- `test_lookup_paper_by_key()` — exact zotero_key match
- `test_lookup_paper_by_doi()` — DOI lookup
- `test_lookup_paper_by_title_substring()` — LIKE match
- `test_lookup_paper_no_results()` — returns empty list
- `test_get_paper_status_returns_none_for_missing()` — paper not found
- `test_entry_from_row_handles_null_fields()` — None values don't crash
---
### WR-05: CLI dispatch pattern inconsistent with existing codebase
**File:** Plan Task 6, Step 3 (cli.py dispatch blocks)
**Issue:** The plan adds verbose-index carving logic in the dispatch blocks:
```python
if args.command == "memory":
argv = sys.argv.copy()
try:
idx = argv.index("memory")
args.verbose = "--verbose" in argv[idx:] or "-v" in argv[idx:]
except ValueError:
pass
from paperforge.commands import memory
return memory.run(args)
```
No other command dispatch in `cli.py` (lines 407-533) uses this pattern. All 15 existing command dispatches simply import and call `run(args)`. The `--verbose` flag is already a top-level argument parsed by argparse (cli.py lines 132-136), and `configure_logging(verbose=...)` is called at line 402 BEFORE any dispatch. This carving code is redundant and adds 14 lines of unnecessary complexity per command.
**Fix:** Follow the existing pattern — just import and dispatch:
```python
if args.command == "memory":
from paperforge.commands import memory
return memory.run(args)
if args.command == "paper-status":
from paperforge.commands import paper_status
return paper_status.run(args)
```
---
## Info
### IN-01: Unused `compute_health` import in `builder.py`
**File:** Plan Task 4, Step 1 (`paperforge/memory/builder.py`, line 414)
**Issue:** `compute_health` is imported but never called in `build_from_index()`. Per the spec (line 141), health dimensions are computed at query time only, so this import is conceptually correct to exclude. The dead import is harmless but clutters the import block.
**Fix:** Remove `compute_health` from the builder import:
```python
from paperforge.worker.asset_state import (
compute_lifecycle,
compute_maturity,
compute_next_step,
)
```
---
### IN-02: `_COMMAND_REGISTRY` entries not consumed by `cli.py` dispatch
**File:** Plan Task 6, Step 4 (`paperforge/commands/__init__.py`)
**Issue:** The plan adds `"memory"` and `"paper-status"` to `_COMMAND_REGISTRY`, which powers `get_command_module()` for dynamic dispatch. However, `cli.py` uses hard-coded `if/elif` chains (not `get_command_module()`), so these registry entries are unused by the primary dispatch path. The entries are only consumed if some other code path calls `get_command_module("memory")`.
**Fix:** Not critical for Phase 1, but either (a) use `get_command_module()` in cli.py dispatch to reduce duplication, or (b) document that the registry exists for future dynamic-dispatch migration.
---
### IN-03: `_compute_hash` uses `.get()` instead of direct key access per spec
**File:** Plan Task 4, Step 1 (`paperforge/memory/builder.py`, line 448-449)
**Issue:** The spec (line 202) explicitly says:
> `sorted(items, key=lambda e: e["zotero_key"])`
The plan uses `e.get("zotero_key", "")` — a safe-access variant. This is arguably more robust (it won't crash on malformed entries), but the spec's direct-access was an intentional design choice to fail-loud on corrupt data rather than silently producing a different hash. Decide which contract you want.
**Fix:** Either align with spec (remove `.get()` for loud failure) or update the spec to accept safe access.
---
### IN-04: `_entry_from_row` uses fragile `.rstrip("_json")`
**File:** Plan Task 5, Step 1 (`paperforge/memory/query.py`, line 729)
**Issue:**
```python
entry[key.rstrip("_json")] = json.loads(entry.pop(key))
```
`rstrip("_json")` removes any trailing characters in the set `{'_', 'j', 's', 'o', 'n'}`, not the literal substring `"_json"`. For `"authors_json"` this produces `"authors"` (correct), and for `"collections_json"` it produces `"collections"` (correct). But if future columns with names like `"version_json"` or `"annotation_json"` were added, this would produce `"versi"` or `"annotati"` — silently wrong. The fix is trivial and prevents future bugs.
**Fix:**
```python
for key in ("authors_json", "collections_json"):
if key in entry and entry[key]:
try:
clean_key = key[:-5] # strip "_json" suffix (exactly 5 chars)
entry[clean_key] = json.loads(entry.pop(key))
except json.JSONDecodeError:
pass
```
---
_Reviewed: 2026-05-12T18:30:00Z_
_Reviewer: VT-OS/OPENCODE (gsd-code-reviewer)_
_Depth: deep_

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,279 @@
# Memory Layer — Design Spec
> **Status:** Approved | **Date:** 2026-05-12
> **Review:** Passed (v2 — 5 BLOCKER, 3 MAJOR, 6 MINOR resolved)
## Goal
Add a SQLite-backed Memory Layer to PaperForge as a derived, rebuildable global index that serves
dashboard, resolver, agent-context, and search commands.
## Architecture
```
Zotero/BetterBibTeX → exports/*.json
formal-library.json (Canonical Index — source of truth, already exists)
paperforge.db (Memory Layer — derived, rebuildable SQLite index)
paper-status / dashboard / agent-context / search / retrieve
```
**Core principle:** `paperforge.db` is a derived index, not the source of truth.
It can be safely deleted and rebuilt from `formal-library.json` at any time.
## Phase 1 Scope
**Tables:** `meta`, `papers`, `paper_assets`, `paper_aliases`
**Commands:**
- `paperforge memory build --json`
- `paperforge memory status --json`
- `paperforge paper-status <query> --json`
**NOT in Phase 1:** FTS5, chunk retrieval, embedding, `paperforge.db → Markdown` writes,
agent-context, dashboard integration.
## SQLite Location
```
<system_dir>/PaperForge/indexes/paperforge.db
```
(same directory as `formal-library.json`)
Register a new path key `"memory_db"` in `config.py:paperforge_paths()` pointing to
`paperforge / "indexes" / "paperforge.db"`. Do not reuse the existing `"index"` key.
## Schema
### Connection settings
- `PRAGMA journal_mode=WAL;` — allow concurrent reads during rebuild
- `PRAGMA foreign_keys=ON;`
### meta
```sql
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
```
Stores: `schema_version` (integer), `paperforge_version`, `created_at`, `last_full_build_at`,
`canonical_index_hash`, `canonical_index_generated_at`.
### Schema versioning strategy
On `paperforge memory build`, if the stored `schema_version` in `meta` does not match the
current version, DROP all tables and rebuild from scratch. `paperforge.db` is a derived index
— full rebuild is always safe. This mirrors `formal-library.json`'s schema-version-check
pattern in `asset_index.py:475-480`.
Initial schema version: `1`.
### papers
One row per paper. Columns directly map to `_build_entry()` entry dict fields.
`asset_state.py` pure functions (`compute_lifecycle`, `compute_health`, `compute_maturity`,
`compute_next_step`) are called at **build time** on each entry dict to populate derived columns.
```sql
CREATE TABLE IF NOT EXISTS papers (
zotero_key TEXT PRIMARY KEY,
citation_key TEXT NOT NULL DEFAULT '',
title TEXT NOT NULL,
year TEXT,
doi TEXT,
pmid TEXT,
journal TEXT,
first_author TEXT,
authors_json TEXT, -- json.dumps(entry["authors"], ensure_ascii=False)
abstract TEXT,
domain TEXT,
collection_path TEXT,
collections_json TEXT, -- json.dumps(entry["collections"], ensure_ascii=False)
has_pdf INTEGER NOT NULL DEFAULT 0,
do_ocr INTEGER,
analyze INTEGER,
ocr_status TEXT,
deep_reading_status TEXT,
ocr_job_id TEXT,
impact_factor REAL,
lifecycle TEXT, -- compute_lifecycle(entry) → "indexed"|"pdf_ready"|"fulltext_ready"|"deep_read_done"
maturity_level INTEGER, -- compute_maturity(entry)["level"] → 1-4
maturity_name TEXT, -- compute_maturity(entry)["level_name"]
next_step TEXT, -- compute_next_step(entry) → "sync"|"ocr"|"/pf-deep"|"ready"
pdf_path TEXT,
note_path TEXT,
main_note_path TEXT,
paper_root TEXT,
fulltext_path TEXT,
ocr_md_path TEXT,
ocr_json_path TEXT,
ai_path TEXT,
deep_reading_md_path TEXT,
updated_at TEXT -- envelope["generated_at"] from formal-library.json
);
```
Indexes:
```sql
CREATE INDEX IF NOT EXISTS idx_papers_zotero_key ON papers(zotero_key);
CREATE INDEX IF NOT EXISTS idx_papers_citation_key ON papers(citation_key);
CREATE INDEX IF NOT EXISTS idx_papers_doi ON papers(doi);
CREATE INDEX IF NOT EXISTS idx_papers_domain ON papers(domain);
CREATE INDEX IF NOT EXISTS idx_papers_year ON papers(year);
CREATE INDEX IF NOT EXISTS idx_papers_ocr_status ON papers(ocr_status);
CREATE INDEX IF NOT EXISTS idx_papers_deep_status ON papers(deep_reading_status);
CREATE INDEX IF NOT EXISTS idx_papers_lifecycle ON papers(lifecycle);
CREATE INDEX IF NOT EXISTS idx_papers_next_step ON papers(next_step);
```
**Important notes about column→entry mapping:**
- `maturity_level` = `compute_maturity(entry)["level"]` (scalar 1-4, not the full dict)
- `updated_at` = the envelope's `generated_at` timestamp from `formal-library.json` (shared across all papers in a build)
- `lifecycle` values: `"indexed"`, `"pdf_ready"`, `"fulltext_ready"`, `"deep_read_done"` — these are NOT all members of the `Lifecycle` enum in `core/state.py` (which has `OCR_READY`, `ANALYZE_READY`, `ERROR_STATE` that are never produced). Use plain string comparison, not enum membership.
- `ai_context_ready` is a pre-seeded zero in `summarize_index()` (`asset_index.py:644`) but is never produced by `compute_lifecycle()`. Keep the zero bucket for Phase 3 compatibility but document it as reserved.
**Health dimensions** (`pdf_health`, `ocr_health`, `note_health`, `asset_health`) are NOT stored in the papers table. They are computed at query time via `asset_state.compute_health(entry_dict)`. The `paper-status` command reconstructs the entry dict from SQLite columns, then calls `compute_health()` in-process.
### paper_assets
```sql
CREATE TABLE IF NOT EXISTS paper_assets (
paper_id TEXT NOT NULL,
asset_type TEXT NOT NULL,
path TEXT NOT NULL,
exists_on_disk INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (paper_id, asset_type),
FOREIGN KEY (paper_id) REFERENCES papers(zotero_key)
);
```
Asset types and their source fields:
| asset_type | source in entry dict | notes |
| -------------- | -------------------------- | --------------------------------------------------------- |
| `pdf` | `pdf_path` | wiki-link; check existence via filesystem |
| `formal_note` | `note_path` | relative vault path |
| `main_note` | `main_note_path` | workspace `{key}.md` |
| `ocr_fulltext` | `fulltext_path` | copied from `ocr/{key}/fulltext.md` |
| `ocr_meta` | derived from `ocr_json_path` | `ocr/{key}/meta.json` |
| `deep_reading` | `main_note_path` | checks for `## 🔍 精读` section within main note (NOT a separate file; `deep_reading_path` is deprecated and always empty) |
| `ai_dir` | `ai_path` | workspace `ai/` directory |
### paper_aliases
```sql
CREATE TABLE IF NOT EXISTS paper_aliases (
id INTEGER PRIMARY KEY AUTOINCREMENT,
paper_id TEXT NOT NULL,
alias TEXT NOT NULL,
alias_norm TEXT NOT NULL,
alias_type TEXT NOT NULL,
FOREIGN KEY (paper_id) REFERENCES papers(zotero_key)
);
```
Alias types (Phase 1):
| alias_type | source | normalized to lowercase |
| ------------- | ----------------- | --------------------------- |
| `zotero_key` | `entry["zotero_key"]` | as-is (uppercase) |
| `citation_key` | `entry["citation_key"]` | as-is (case-sensitive) |
| `title` | `entry["title"]` | `.lower().strip()` |
| `doi` | `entry["doi"]` | `.lower().strip()` |
## Commands
### `paperforge memory build --json`
1. Resolve vault path
2. Read `formal-library.json` (canonical index envelope) via `read_index(vault)`
3. If index is `None` or missing → return `PFResult(ok=False, error=PFError(code=PATH_NOT_FOUND, message="Canonical index not found. Run paperforge sync --rebuild-index."))`
4. Extract `items` list and envelope metadata
5. Create/open `paperforge.db` (WAL mode)
6. If stored `schema_version` != current → DROP all tables
7. Create tables if not exist
8. Upsert `meta` rows: `schema_version`, `paperforge_version`, `created_at`, `last_full_build_at`, `canonical_index_generated_at`
9. Compute `canonical_index_hash` = SHA-256 of `json.dumps(sorted(items, key=lambda e: e["zotero_key"]), sort_keys=True, ensure_ascii=False)`; store in `meta`
10. For each entry in `items`:
- Insert/upsert into `papers`
- Insert/upsert into `paper_assets` (check `exists_on_disk` via `Path.exists()`)
- Insert/upsert into `paper_aliases`
11. Return `PFResult(ok=True, data={...})` with `papers_indexed`, `assets_indexed`, `aliases_indexed` counts
**PFResult.next_actions format** (must match `core/result.py:26``list[dict]`):
```json
{
"next_actions": [
{"command": "paperforge paper-status <key> --json", "reason": "Look up a specific paper"}
]
}
```
### `paperforge memory status --json`
Check:
- `paperforge.db` exists → `db_exists: bool`
- `schema_version` matches current → `schema_ok: bool`
- `canonical_index_hash` matches computed hash of current `formal-library.json``fresh: bool`
- Paper count matches `envelope["paper_count"]``count_match: bool`
- Any check fails → `needs_rebuild: true`
Return `PFResult(ok=True, data={...})`.
### `paperforge paper-status <query> --json`
**Resolution is short-circuit:** stop at the first step that returns ≥1 result.
Resolution order:
1. Exact match on `zotero_key` (case-insensitive)
2. Exact match on `citation_key` (case-insensitive)
3. Exact match on `doi` (case-insensitive)
4. LIKE match on `title_norm` or normalized alias (`%<query_lower>%`)
5. Fallback: search `paper_aliases.alias_norm`
Behavior by result count:
- **0 results:** `PFResult(ok=False, error=NOT_FOUND, next_actions=[{"command": "paperforge search", ...}])`
- **1 result:** Full status with paper metadata, assets, lifecycle, next_step, recommended action
- **>1 results:** Candidate list only (no full status details)
Full status response includes:
- Paper metadata (title, year, authors, doi, journal, domain, abstract)
- Asset status (exists_on_disk for each asset type)
- Lifecycle state
- Health dimensions (computed at query time via `compute_health()`)
- Maturity level and name
- `next_step` with recommended action
- `recommended_action`: e.g., `"/pf-deep ABCDEFG"` or `"paperforge sync"` or `"paperforge ocr"`
## Integration Points
### With sync
After `paperforge sync` completes, optionally refresh memory for changed keys.
Not automatic in Phase 1.
### With dashboard
Dashboard should prefer `paperforge.db` for stats, fallback to file scanning.
This integration is deferred to Phase 2.
### With agent
Agent skill bootstrap runs `paperforge agent-context --compact --json` first.
This command is deferred to Phase 3.
## Constraints
1. `paperforge.db` is a derived index — deletable, rebuildable
2. No SQLite → Markdown writes in Phase 1
3. Reuse `asset_state.py` pure functions (compute_lifecycle, compute_health, compute_maturity, compute_next_step)
4. Health dimensions are computed at query time via `compute_health()`, not stored in SQLite
5. All `--json` output uses PFResult envelope (respecting `next_actions: list[dict]` contract)
6. SQLite connection uses WAL mode for concurrent reads
7. No external database services — only Python stdlib `sqlite3`
8. No PDF/image binary storage
9. No embedding or vector DB
10. Schema version mismatch → full drop-and-rebuild (derived index, always safe)

View file

@ -1,7 +1,7 @@
{
"id": "paperforge",
"name": "PaperForge",
"version": "1.5.5",
"version": "1.5.6rc1",
"minAppVersion": "1.9.0",
"description": "PaperForge — Zotero literature pipeline. Sync PDFs, run OCR, and read with AI-assisted deep reading.",
"author": "Lin Zhaoxuan",

View file

@ -1,3 +1,3 @@
"""paperforge — PaperForge package."""
__version__ = "1.5.5"
__version__ = "1.5.6rc1"

View file

@ -1,7 +1,7 @@
{
"id": "paperforge",
"name": "PaperForge",
"version": "1.5.5",
"version": "1.5.6rc1",
"minAppVersion": "1.9.0",
"description": "PaperForge — Zotero literature pipeline. Sync PDFs, run OCR, and read with AI-assisted deep reading.",
"author": "Lin Zhaoxuan",