mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
49f84a1aad
Rewrite embedding provider to use openai.OpenAI client instead of raw requests.post. Old code preserved in requests_fallback.py, selectable via VECTOR_DB_PROVIDER_TYPE env/setting. Changes: - paperforge/embedding/providers/openai_compatible.py: rewrite to SDK - paperforge/embedding/providers/requests_fallback.py: NEW old impl - paperforge/embedding/_config.py: add get_provider_type() - tests/test_openai_compatible.py: 6 new tests (encode, timeout, provider_type switch, no-key error, custom base url) feat(#33): E2E test fixtures — 3 synthetic PDFs + expected outputs 3 synthetic papers generated with PyMuPDF, each 3 pages: - paper_a: The Effect of Machine Learning on Clinical Outcomes - paper_b: A Randomized Trial of Remimazolam vs Propofol (with tables) - paper_c: Deep Learning for Medical Image Segmentation (with figures) Changes: - tests/fixtures/papers/: 3 PDFs - tests/fixtures/expected_outputs/: expected blocks + body units - tests/conftest.py: e2e_fixture_dir + synthetic_paper_paths fixtures feat(#26): sqlite-vec schema — vec0 virtual tables + build_state Schema version bumped 4→5. Adds: - 3 vec0 virtual tables: vec_fulltext, vec_body, vec_objects (1536-dim) - 3 companion metadata tables: vec_*_meta (paper_id, chunk_index, text) - build_state table (key/value/updated_at) - ensure_vec_extension() in db.py for extension loading Changes: - paperforge/memory/schema.py: schema v5, new tables in ensure_schema() - paperforge/memory/db.py: ensure_vec_extension() helper - tests/test_vector_schema.py: 5 new tests (tables, virtual, idempotent, v5)
97 lines
2.7 KiB
Python
97 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
|
|
from paperforge.memory.schema import (
|
|
ALL_TABLES,
|
|
CURRENT_SCHEMA_VERSION,
|
|
ensure_schema,
|
|
get_schema_version,
|
|
)
|
|
|
|
|
|
def _vec_available(conn: sqlite3.Connection) -> bool:
|
|
"""Return True if sqlite-vec extension can be loaded."""
|
|
try:
|
|
conn.enable_load_extension(True)
|
|
conn.load("vec0")
|
|
conn.enable_load_extension(False)
|
|
return True
|
|
except (sqlite3.OperationalError, AttributeError):
|
|
return False
|
|
|
|
|
|
def test_ensure_schema_creates_vec_tables():
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.row_factory = sqlite3.Row
|
|
ensure_schema(conn)
|
|
|
|
cursor = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
|
)
|
|
tables = {row["name"] for row in cursor.fetchall()}
|
|
|
|
# Companion tables + build_state always created (plain tables)
|
|
assert "vec_fulltext_meta" in tables
|
|
assert "vec_body_meta" in tables
|
|
assert "vec_objects_meta" in tables
|
|
assert "build_state" in tables
|
|
|
|
# Vec0 virtual tables only created when extension is available
|
|
if _vec_available(conn):
|
|
assert "vec_fulltext" in tables
|
|
assert "vec_body" in tables
|
|
assert "vec_objects" in tables
|
|
else:
|
|
assert "vec_fulltext" not in tables
|
|
assert "vec_body" not in tables
|
|
assert "vec_objects" not in tables
|
|
conn.close()
|
|
|
|
|
|
def test_vec_tables_are_virtual():
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.row_factory = sqlite3.Row
|
|
ensure_schema(conn)
|
|
|
|
if not _vec_available(conn):
|
|
conn.close()
|
|
return # Skip virtual-table check when extension not loaded
|
|
|
|
cursor = conn.execute(
|
|
"SELECT name, type FROM sqlite_master WHERE type='virtual' ORDER BY name"
|
|
)
|
|
virtual_tables = {row["name"] for row in cursor.fetchall()}
|
|
assert "vec_fulltext" in virtual_tables
|
|
assert "vec_body" in virtual_tables
|
|
assert "vec_objects" in virtual_tables
|
|
conn.close()
|
|
|
|
|
|
def test_build_state_table_exists():
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.row_factory = sqlite3.Row
|
|
ensure_schema(conn)
|
|
|
|
cursor = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' AND name='build_state'"
|
|
)
|
|
assert cursor.fetchone() is not None
|
|
conn.close()
|
|
|
|
|
|
def test_schema_creation_is_idempotent():
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.row_factory = sqlite3.Row
|
|
ensure_schema(conn)
|
|
ensure_schema(conn) # second call must not raise
|
|
conn.close()
|
|
|
|
|
|
def test_schema_version_is_5():
|
|
conn = sqlite3.connect(":memory:")
|
|
conn.row_factory = sqlite3.Row
|
|
ensure_schema(conn)
|
|
assert get_schema_version(conn) == 5
|
|
assert CURRENT_SCHEMA_VERSION == 5
|
|
conn.close()
|