mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 17:00:23 +00:00
448 lines
17 KiB
Python
448 lines
17 KiB
Python
"""Tests for asset_index.py — envelope format, atomic writes, locking.
|
|
|
|
Covers: get_index_path, build_envelope, atomic_write_index, build_index
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import threading
|
|
from pathlib import Path
|
|
|
|
import filelock
|
|
import pytest
|
|
|
|
from paperforge.worker.asset_index import (
|
|
CURRENT_SCHEMA_VERSION,
|
|
INDEX_FILENAME,
|
|
atomic_write_index,
|
|
build_envelope,
|
|
build_index,
|
|
get_index_path,
|
|
summarize_index,
|
|
)
|
|
from paperforge.worker.sync import frontmatter_note
|
|
|
|
|
|
class TestGetIndexPath:
|
|
"""get_index_path(vault) -> Path"""
|
|
|
|
def test_returns_path_suffix(self, tmp_path: Path) -> None:
|
|
"""Index path must end with indexes/formal-library.json."""
|
|
vault = _minimal_vault(tmp_path)
|
|
result = get_index_path(vault)
|
|
assert isinstance(result, Path)
|
|
assert result.name == INDEX_FILENAME
|
|
assert result.parent.name == "indexes"
|
|
|
|
def test_path_uses_paperforge_dir(self, tmp_path: Path) -> None:
|
|
"""Index path is inside the PaperForge system directory."""
|
|
vault = _minimal_vault(tmp_path)
|
|
result = get_index_path(vault)
|
|
assert "PaperForge" in result.parts
|
|
assert "indexes" in result.parts
|
|
|
|
|
|
class TestBuildEnvelope:
|
|
"""build_envelope(items) -> dict"""
|
|
|
|
def test_empty_list(self) -> None:
|
|
"""Empty items list produces schema_version='2', paper_count=0."""
|
|
envelope = build_envelope([])
|
|
assert envelope["schema_version"] == CURRENT_SCHEMA_VERSION
|
|
assert envelope["paper_count"] == 0
|
|
assert envelope["items"] == []
|
|
assert "generated_at" in envelope
|
|
assert envelope["generated_at"].endswith("+08:00") or envelope["generated_at"].endswith("Z")
|
|
|
|
def test_with_items(self) -> None:
|
|
"""Envelope with 3 items has matching paper_count."""
|
|
items = [
|
|
{"zotero_key": "AAA", "title": "Paper A"},
|
|
{"zotero_key": "BBB", "title": "Paper B"},
|
|
{"zotero_key": "CCC", "title": "Paper C"},
|
|
]
|
|
envelope = build_envelope(items)
|
|
assert envelope["paper_count"] == 3
|
|
assert envelope["items"] == items
|
|
assert envelope["schema_version"] == CURRENT_SCHEMA_VERSION
|
|
|
|
def test_paper_count_matches_length(self) -> None:
|
|
"""paper_count always equals len(items) regardless of content."""
|
|
for n in (0, 1, 5, 100):
|
|
items = [{"id": i} for i in range(n)]
|
|
envelope = build_envelope(items)
|
|
assert envelope["paper_count"] == n
|
|
assert len(envelope["items"]) == n
|
|
|
|
def test_generated_at_is_beijing_time(self) -> None:
|
|
"""generated_at ends with +08:00 (Beijing time) and is parseable."""
|
|
envelope = build_envelope([])
|
|
ts = envelope["generated_at"]
|
|
assert ts.endswith("+08:00")
|
|
# Should be ISO-format date (rough check): YYYY-MM-DDT...
|
|
assert "T" in ts
|
|
|
|
|
|
class TestAtomicWriteIndex:
|
|
"""atomic_write_index(path, data) -> None"""
|
|
|
|
def test_creates_file(self, tmp_path: Path) -> None:
|
|
"""Writes valid JSON to the target file."""
|
|
target = tmp_path / "indexes" / "formal-library.json"
|
|
data = {"schema_version": "2", "paper_count": 0, "items": []}
|
|
atomic_write_index(target, data)
|
|
assert target.exists()
|
|
loaded = json.loads(target.read_text(encoding="utf-8"))
|
|
assert loaded == data
|
|
|
|
def test_content_is_correct(self, tmp_path: Path) -> None:
|
|
"""Written content matches input data exactly."""
|
|
target = tmp_path / "out.json"
|
|
data = {"items": [{"key": "ABC"}], "paper_count": 1}
|
|
atomic_write_index(target, data)
|
|
assert json.loads(target.read_text(encoding="utf-8")) == data
|
|
|
|
def test_no_partial_file_on_interrupt(self, monkeypatch, tmp_path: Path) -> None:
|
|
"""Simulate a crash after NamedTemporaryFile write but before os.replace.
|
|
|
|
The target file should either contain the old content or not exist
|
|
— never a half-written partial file.
|
|
"""
|
|
target = tmp_path / "safe.json"
|
|
original_replace = os.replace
|
|
calls = []
|
|
|
|
def _failing_replace(src, dst):
|
|
calls.append((src, dst))
|
|
raise OSError("Simulated crash during replace")
|
|
|
|
monkeypatch.setattr(os, "replace", _failing_replace)
|
|
|
|
with pytest.raises(OSError, match="Simulated crash"):
|
|
atomic_write_index(target, {"key": "should-not-appear"})
|
|
|
|
# Temp file should have been cleaned up
|
|
temp_files = list(tmp_path.glob("*.tmp"))
|
|
assert len(temp_files) == 0, f"Temp file not cleaned up: {temp_files}"
|
|
|
|
# Target file should not exist (never completed os.replace)
|
|
assert not target.exists() or target.read_text(encoding="utf-8") == ""
|
|
|
|
def test_creates_parent_dirs(self, tmp_path: Path) -> None:
|
|
"""Creates intermediate directories automatically."""
|
|
target = tmp_path / "a" / "b" / "c" / "index.json"
|
|
atomic_write_index(target, {"key": "val"})
|
|
assert target.exists()
|
|
assert json.loads(target.read_text(encoding="utf-8")) == {"key": "val"}
|
|
|
|
def test_overwrites_existing(self, tmp_path: Path) -> None:
|
|
"""Overwrites an existing file atomically."""
|
|
target = tmp_path / "existing.json"
|
|
target.write_text(json.dumps({"old": "data"}), encoding="utf-8")
|
|
atomic_write_index(target, {"new": "data"})
|
|
assert json.loads(target.read_text(encoding="utf-8")) == {"new": "data"}
|
|
|
|
def test_lock_timeout(self, tmp_path: Path) -> None:
|
|
"""Second concurrent writer raises filelock.Timeout."""
|
|
target = tmp_path / "locked.json"
|
|
lock_path = target.with_suffix(".json.lock")
|
|
held = threading.Event()
|
|
done = threading.Event()
|
|
|
|
def _holder():
|
|
lock = filelock.FileLock(lock_path, timeout=30)
|
|
with lock:
|
|
held.set()
|
|
# Hold the lock longer than LOCK_TIMEOUT
|
|
done.wait(timeout=15)
|
|
|
|
t = threading.Thread(target=_holder, daemon=True)
|
|
t.start()
|
|
held.wait(timeout=5) # Wait for holder to acquire
|
|
|
|
# Now try to write — should time out
|
|
with pytest.raises(filelock.Timeout):
|
|
atomic_write_index(target, {"will": "timeout"})
|
|
|
|
done.set()
|
|
t.join(timeout=5)
|
|
|
|
|
|
class TestBuildIndexEmpty:
|
|
"""build_index(vault, verbose=False) with empty exports"""
|
|
|
|
def test_empty_exports_dir(self, tmp_path: Path) -> None:
|
|
"""Empty exports directory returns 0 and writes empty envelope."""
|
|
vault = _minimal_vault(tmp_path)
|
|
|
|
# Create empty exports directory
|
|
exports_dir = vault / "99_System" / "PaperForge" / "exports"
|
|
exports_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create domain config so load_domain_config doesn't crash
|
|
_ensure_domain_config(vault)
|
|
|
|
count = build_index(vault, verbose=False)
|
|
assert count == 0
|
|
|
|
# Verify empty envelope was written
|
|
index_path = get_index_path(vault)
|
|
assert index_path.exists()
|
|
envelope = json.loads(index_path.read_text(encoding="utf-8"))
|
|
assert envelope["schema_version"] == CURRENT_SCHEMA_VERSION
|
|
assert envelope["paper_count"] == 0
|
|
assert envelope["items"] == []
|
|
|
|
def test_absent_exports_dir(self, tmp_path: Path) -> None:
|
|
"""No exports directory returns 0 and writes empty envelope."""
|
|
vault = _minimal_vault(tmp_path)
|
|
|
|
# Do NOT create exports dir — it should not exist
|
|
_ensure_domain_config(vault)
|
|
|
|
count = build_index(vault, verbose=False)
|
|
assert count == 0
|
|
|
|
index_path = get_index_path(vault)
|
|
assert index_path.exists()
|
|
envelope = json.loads(index_path.read_text(encoding="utf-8"))
|
|
assert envelope["paper_count"] == 0
|
|
|
|
def test_build_index_does_not_auto_embed_vectors(self, monkeypatch, tmp_path: Path) -> None:
|
|
"""sync/index build must not trigger vector embedding work."""
|
|
vault = _minimal_vault(tmp_path)
|
|
_ensure_domain_config(vault)
|
|
|
|
exports_dir = vault / "99_System" / "PaperForge" / "exports"
|
|
exports_dir.mkdir(parents=True, exist_ok=True)
|
|
(exports_dir / "library.json").write_text(
|
|
json.dumps([
|
|
{
|
|
"key": "ABC12345",
|
|
"title": "Paper A",
|
|
"creators": [],
|
|
"collections": [],
|
|
"attachments": [],
|
|
"doi": "",
|
|
"pmid": "",
|
|
"date": "",
|
|
"extra": "",
|
|
"abstractNote": "",
|
|
"publicationTitle": "",
|
|
"itemType": "journalArticle",
|
|
}
|
|
]),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _unexpected_auto_embed(_vault, _entry):
|
|
raise AssertionError("build_index must not auto-embed vectors")
|
|
|
|
monkeypatch.setattr("paperforge.worker.asset_index._vec_auto_embed_if_new", _unexpected_auto_embed)
|
|
|
|
build_index(vault, verbose=False)
|
|
|
|
|
|
class TestSummarizeIndex:
|
|
"""summarize_index(vault) -> dict | None"""
|
|
|
|
def _write_envelope(self, vault: Path, items: list[dict]) -> None:
|
|
"""Helper: write an envelope-format index to the vault."""
|
|
from paperforge.worker.asset_index import atomic_write_index, build_envelope
|
|
|
|
idx_path = get_index_path(vault)
|
|
idx_path.parent.mkdir(parents=True, exist_ok=True)
|
|
envelope = build_envelope(items)
|
|
atomic_write_index(idx_path, envelope)
|
|
|
|
def test_summarize_index_returns_aggregates(self, tmp_path: Path) -> None:
|
|
"""3 items at different lifecycle states produce correct counts."""
|
|
vault = _minimal_vault(tmp_path)
|
|
items = [
|
|
{
|
|
"zotero_key": "AAA",
|
|
"lifecycle": "indexed",
|
|
"health": {
|
|
"pdf_health": "healthy",
|
|
"ocr_health": "healthy",
|
|
"note_health": "healthy",
|
|
"asset_health": "healthy",
|
|
},
|
|
"maturity": {"level": 1},
|
|
},
|
|
{
|
|
"zotero_key": "BBB",
|
|
"lifecycle": "fulltext_ready",
|
|
"health": {
|
|
"pdf_health": "healthy",
|
|
"ocr_health": "healthy",
|
|
"note_health": "PDF path missing",
|
|
"asset_health": "healthy",
|
|
},
|
|
"maturity": {"level": 3},
|
|
},
|
|
{
|
|
"zotero_key": "CCC",
|
|
"lifecycle": "ai_context_ready",
|
|
"health": {
|
|
"pdf_health": "healthy",
|
|
"ocr_health": "healthy",
|
|
"note_health": "healthy",
|
|
"asset_health": "healthy",
|
|
},
|
|
"maturity": {"level": 6},
|
|
},
|
|
]
|
|
self._write_envelope(vault, items)
|
|
result = summarize_index(vault)
|
|
assert result is not None
|
|
assert result["paper_count"] == 3
|
|
assert result["lifecycle_level_counts"]["indexed"] == 1
|
|
assert result["lifecycle_level_counts"]["fulltext_ready"] == 1
|
|
assert result["lifecycle_level_counts"]["ai_context_ready"] == 1
|
|
# Unhealthy in note_health due to "PDF path missing"
|
|
assert result["health_aggregate"]["note_health"]["unhealthy"] == 1
|
|
assert result["health_aggregate"]["note_health"]["healthy"] == 2
|
|
assert result["maturity_distribution"]["1"] == 1
|
|
assert result["maturity_distribution"]["3"] == 1
|
|
assert result["maturity_distribution"]["6"] == 1
|
|
|
|
def test_summarize_index_missing_index_returns_none(self, tmp_path: Path) -> None:
|
|
"""When index file does not exist, returns None."""
|
|
vault = _minimal_vault(tmp_path)
|
|
# Do NOT write an index file
|
|
result = summarize_index(vault)
|
|
assert result is None
|
|
|
|
def test_summarize_index_legacy_format_returns_none(self, tmp_path: Path) -> None:
|
|
"""When index is a bare list (legacy), returns None."""
|
|
vault = _minimal_vault(tmp_path)
|
|
idx_path = get_index_path(vault)
|
|
idx_path.parent.mkdir(parents=True, exist_ok=True)
|
|
# Legacy format: bare list
|
|
idx_path.write_text(json.dumps([{"zotero_key": "AAA"}]), encoding="utf-8")
|
|
result = summarize_index(vault)
|
|
assert result is None
|
|
|
|
def test_summarize_index_empty_items_all_zeros(self, tmp_path: Path) -> None:
|
|
"""When items list is empty, all counts are zero."""
|
|
vault = _minimal_vault(tmp_path)
|
|
self._write_envelope(vault, [])
|
|
result = summarize_index(vault)
|
|
assert result is not None
|
|
assert result["paper_count"] == 0
|
|
for key in ("indexed", "pdf_ready", "fulltext_ready", "deep_read_done", "ai_context_ready"):
|
|
assert result["lifecycle_level_counts"][key] == 0
|
|
for dim in ("pdf_health", "ocr_health", "note_health", "asset_health"):
|
|
assert result["health_aggregate"][dim]["healthy"] == 0
|
|
assert result["health_aggregate"][dim]["unhealthy"] == 0
|
|
for level in range(1, 7):
|
|
assert result["maturity_distribution"][str(level)] == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _minimal_vault(tmp_path: Path) -> Path:
|
|
"""Create a minimal vault with paperforge.json for path resolution."""
|
|
vault = tmp_path / "test_vault"
|
|
vault.mkdir(parents=True, exist_ok=True)
|
|
pf_json = vault / "paperforge.json"
|
|
pf_json.write_text(
|
|
json.dumps(
|
|
{
|
|
"version": "1.2.0",
|
|
"vault_config": {
|
|
"system_dir": "99_System",
|
|
"resources_dir": "03_Resources",
|
|
"literature_dir": "Literature",
|
|
"control_dir": "LiteratureControl",
|
|
"base_dir": "05_Bases",
|
|
"skill_dir": ".opencode/skills",
|
|
},
|
|
},
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return vault
|
|
|
|
|
|
class TestFrontmatterNoteTagPreservation:
|
|
"""Unit tests for frontmatter_note() tag preservation behavior."""
|
|
|
|
def _make_entry(self, zotero_key: str, domain: str = "test") -> dict:
|
|
return {
|
|
"title": "Test Paper",
|
|
"domain": domain,
|
|
"zotero_key": zotero_key,
|
|
"first_author": "Author",
|
|
"authors": ["Author"],
|
|
"doi": "",
|
|
"pmid": "",
|
|
"year": "",
|
|
"journal": "",
|
|
"citation_key": "",
|
|
"collections": [],
|
|
"collection_tags": [],
|
|
"collection_path": "",
|
|
"impact_factor": "",
|
|
"abstract": "",
|
|
"has_pdf": False,
|
|
"do_ocr": False,
|
|
"ocr_status": "pending",
|
|
"deep_reading_status": "pending",
|
|
"pdf_path": "",
|
|
"fulltext_path": "",
|
|
}
|
|
|
|
def test_frontmatter_note_preserves_custom_tags(self):
|
|
entry = self._make_entry("ABC1")
|
|
existing_text = "---\ntags:\n - mine\n - keep\n---\nmy body\n"
|
|
result = frontmatter_note(entry, existing_text)
|
|
assert " - mine" in result
|
|
assert " - keep" in result
|
|
assert " - 文献阅读" not in result
|
|
|
|
def test_frontmatter_note_adds_defaults_when_no_tags(self):
|
|
entry = self._make_entry("ABC2")
|
|
existing_text = "---\ntitle: Test Paper\n---\nbody\n"
|
|
result = frontmatter_note(entry, existing_text)
|
|
assert " - 文献阅读" in result
|
|
assert " - test" in result
|
|
|
|
def test_frontmatter_note_preserves_empty_tags(self):
|
|
entry = self._make_entry("ABC3")
|
|
existing_text = "---\ntags: []\n---\nbody\n"
|
|
result = frontmatter_note(entry, existing_text)
|
|
assert " - 文献阅读" not in result
|
|
|
|
def test_frontmatter_note_falls_back_when_malformed(self):
|
|
entry = self._make_entry("ABC4")
|
|
existing_text = "---\ntags:\n - foo\nbody without closing frontmatter"
|
|
result = frontmatter_note(entry, existing_text)
|
|
assert " - 文献阅读" in result
|
|
|
|
|
|
def _ensure_domain_config(vault: Path) -> None:
|
|
"""Create domain config so load_domain_config returns a valid configuration."""
|
|
from paperforge.config import paperforge_paths as _pp
|
|
|
|
paths = _pp(vault)
|
|
config_dir = paths["paperforge"] / "config"
|
|
config_dir.mkdir(parents=True, exist_ok=True)
|
|
config_path = config_dir / "domain-collections.json"
|
|
config_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"collections": {},
|
|
"domains": [],
|
|
},
|
|
indent=2,
|
|
),
|
|
encoding="utf-8",
|
|
)
|