mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
0dfbf2a68d
New paperforge_objects collection in ChromaDB for figure/table caption vector search, completing the memory layer retrieval triad: paperforge_fulltext → legacy_chunk paperforge_body → body_unit paperforge_objects → object_unit (NEW) Changes: - manifest.py: compute_object_units_hash(), manifest records both hashes - _chroma.py: _COLLECTION_NAMES includes paperforge_objects - builder.py: get_object_units_for_embedding() + embed_object_units() - status.py: object_chunk_count, total_chunks - state_snapshot.py: write_vector_runtime() extended with new fields - search.py: 3rd collection, dict-based source mapping, object_kind/label - __init__.py: exports for new functions - commands/embed.py: _has_object_units_in_db, body+object structured path routing with dual-hash resume, extended counters - Test fix: test_build_paper_manifest data complete for hash functions Verification: 20 object vectors embedded, all per-paper counts match, status total consistent (58266 = 57435 + 811 + 20), object caption queries return object_unit results via merge_retrieve. Unit tests: 153 pass, 1 skip.
71 lines
2.7 KiB
Python
71 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from paperforge.config import paperforge_paths
|
|
from paperforge.core.io import write_json_atomic
|
|
|
|
|
|
def _snapshot_dir(vault: Path) -> Path:
|
|
paths = paperforge_paths(vault)
|
|
d = paths["paperforge"] / "indexes"
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return d
|
|
|
|
|
|
def write_memory_runtime(vault: Path, *, paper_count_db: int,
|
|
paper_count_index: int, fresh: bool,
|
|
needs_rebuild: bool, last_full_build_at: str,
|
|
schema_version_db: int, fts_ready: bool) -> None:
|
|
snap = {
|
|
"schema_version": 1,
|
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
|
"source_command": "paperforge memory status --json",
|
|
"paper_count_db": paper_count_db,
|
|
"paper_count_index": paper_count_index,
|
|
"fresh": fresh,
|
|
"needs_rebuild": needs_rebuild,
|
|
"last_full_build_at": last_full_build_at,
|
|
"schema_version_db": schema_version_db,
|
|
"fts_ready": fts_ready,
|
|
}
|
|
path = _snapshot_dir(vault) / "memory-runtime-state.json"
|
|
write_json_atomic(path, snap)
|
|
|
|
|
|
def write_vector_runtime(vault: Path, *, enabled: bool, mode: str, model: str,
|
|
deps_installed: bool, deps_missing: list[str] | None,
|
|
py_version: str, db_exists: bool, chunk_count: int,
|
|
build_state: dict | None, healthy: bool = True,
|
|
error: str = "", corrupted: bool = False,
|
|
body_chunk_count: int = 0,
|
|
object_chunk_count: int = 0,
|
|
total_chunks: int | None = None) -> None:
|
|
snap = {
|
|
"schema_version": 1,
|
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
|
"source_command": "paperforge embed status --json",
|
|
"enabled": enabled,
|
|
"mode": mode,
|
|
"model": model,
|
|
"deps_installed": deps_installed,
|
|
"deps_missing": deps_missing or [],
|
|
"py_version": py_version,
|
|
"db_exists": db_exists,
|
|
"chunk_count": chunk_count,
|
|
"body_chunk_count": body_chunk_count,
|
|
"object_chunk_count": object_chunk_count,
|
|
"total_chunks": total_chunks if total_chunks is not None else chunk_count + body_chunk_count + object_chunk_count,
|
|
"healthy": healthy,
|
|
"corrupted": corrupted,
|
|
"error": error,
|
|
"build_state": build_state or {},
|
|
}
|
|
path = _snapshot_dir(vault) / "vector-runtime-state.json"
|
|
write_json_atomic(path, snap)
|
|
|
|
|
|
def write_runtime_health(vault: Path, health_data: dict) -> None:
|
|
path = _snapshot_dir(vault) / "runtime-health.json"
|
|
write_json_atomic(path, health_data)
|