mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
0dfbf2a68d
New paperforge_objects collection in ChromaDB for figure/table caption vector search, completing the memory layer retrieval triad: paperforge_fulltext → legacy_chunk paperforge_body → body_unit paperforge_objects → object_unit (NEW) Changes: - manifest.py: compute_object_units_hash(), manifest records both hashes - _chroma.py: _COLLECTION_NAMES includes paperforge_objects - builder.py: get_object_units_for_embedding() + embed_object_units() - status.py: object_chunk_count, total_chunks - state_snapshot.py: write_vector_runtime() extended with new fields - search.py: 3rd collection, dict-based source mapping, object_kind/label - __init__.py: exports for new functions - commands/embed.py: _has_object_units_in_db, body+object structured path routing with dual-hash resume, extended counters - Test fix: test_build_paper_manifest data complete for hash functions Verification: 20 object vectors embedded, all per-paper counts match, status total consistent (58266 = 57435 + 811 + 20), object caption queries return object_unit results via merge_retrieve. Unit tests: 153 pass, 1 skip.
46 lines
1.3 KiB
Python
46 lines
1.3 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
_COLLECTION_NAMES = ["paperforge_fulltext", "paperforge_body", "paperforge_objects"]
|
|
|
|
|
|
def get_vector_db_path(vault: Path) -> Path:
|
|
from paperforge.config import paperforge_paths
|
|
paths = paperforge_paths(vault)
|
|
return (paths.get("memory_db", paths.get("index", vault / "System" / "PaperForge"))).parent / "vectors"
|
|
|
|
|
|
def _get_chroma():
|
|
import chromadb
|
|
return chromadb
|
|
|
|
|
|
def get_collection(vault: Path, name: str = "paperforge_fulltext"):
|
|
chroma = _get_chroma()
|
|
db_path = get_vector_db_path(vault)
|
|
db_path.mkdir(parents=True, exist_ok=True)
|
|
client = chroma.PersistentClient(path=str(db_path))
|
|
return client.get_or_create_collection(
|
|
name=name,
|
|
metadata={"hnsw:space": "cosine"},
|
|
)
|
|
|
|
|
|
def delete_paper_vectors(vault: Path, zotero_key: str) -> int:
|
|
total = 0
|
|
for col_name in _COLLECTION_NAMES:
|
|
try:
|
|
collection = get_collection(vault, name=col_name)
|
|
results = collection.get(where={"paper_id": zotero_key})
|
|
ids = results.get("ids", [])
|
|
if ids:
|
|
collection.delete(ids=ids)
|
|
total += len(ids)
|
|
except Exception:
|
|
pass
|
|
return total
|