mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 17:00:23 +00:00
Rewrite embedding provider to use openai.OpenAI client instead of raw requests.post. Old code preserved in requests_fallback.py, selectable via VECTOR_DB_PROVIDER_TYPE env/setting. Changes: - paperforge/embedding/providers/openai_compatible.py: rewrite to SDK - paperforge/embedding/providers/requests_fallback.py: NEW old impl - paperforge/embedding/_config.py: add get_provider_type() - tests/test_openai_compatible.py: 6 new tests (encode, timeout, provider_type switch, no-key error, custom base url) feat(#33): E2E test fixtures — 3 synthetic PDFs + expected outputs 3 synthetic papers generated with PyMuPDF, each 3 pages: - paper_a: The Effect of Machine Learning on Clinical Outcomes - paper_b: A Randomized Trial of Remimazolam vs Propofol (with tables) - paper_c: Deep Learning for Medical Image Segmentation (with figures) Changes: - tests/fixtures/papers/: 3 PDFs - tests/fixtures/expected_outputs/: expected blocks + body units - tests/conftest.py: e2e_fixture_dir + synthetic_paper_paths fixtures feat(#26): sqlite-vec schema — vec0 virtual tables + build_state Schema version bumped 4→5. Adds: - 3 vec0 virtual tables: vec_fulltext, vec_body, vec_objects (1536-dim) - 3 companion metadata tables: vec_*_meta (paper_id, chunk_index, text) - build_state table (key/value/updated_at) - ensure_vec_extension() in db.py for extension loading Changes: - paperforge/memory/schema.py: schema v5, new tables in ensure_schema() - paperforge/memory/db.py: ensure_vec_extension() helper - tests/test_vector_schema.py: 5 new tests (tables, virtual, idempotent, v5)
198 lines
6.6 KiB
Python
198 lines
6.6 KiB
Python
"""Test fixtures and helpers for PaperForge smoke tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
import sys
|
|
from collections.abc import Generator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
SANDBOX_DIR = REPO_ROOT / "tests" / "sandbox"
|
|
FIXTURE_VAULT = SANDBOX_DIR / "00_TestVault"
|
|
OCR_FIXTURE = SANDBOX_DIR / "ocr-complete" / "TSTONE001"
|
|
EXPORT_FIXTURE = SANDBOX_DIR / "exports" / "骨科.json"
|
|
|
|
|
|
def create_test_vault() -> Path:
|
|
"""Create a fresh test vault with necessary structure."""
|
|
vault = FIXTURE_VAULT
|
|
if vault.exists():
|
|
shutil.rmtree(vault)
|
|
vault.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create directory structure
|
|
system_dir = vault / "99_System"
|
|
pf_dir = system_dir / "PaperForge"
|
|
exports_dir = pf_dir / "exports"
|
|
ocr_dir = pf_dir / "ocr"
|
|
resources_dir = vault / "03_Resources"
|
|
literature_dir = resources_dir / "Literature"
|
|
control_dir = resources_dir / "LiteratureControl"
|
|
records_dir = control_dir / "library-records"
|
|
base_dir = vault / "05_Bases"
|
|
skill_dir = vault / ".opencode" / "skills" / "literature-qa" / "scripts"
|
|
|
|
for d in [exports_dir, ocr_dir, literature_dir, records_dir, base_dir, skill_dir]:
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create paperforge.json
|
|
pf_json = vault / "paperforge.json"
|
|
pf_json.write_text(
|
|
json.dumps(
|
|
{
|
|
"version": "1.2.0",
|
|
"system_dir": "99_System",
|
|
"resources_dir": "03_Resources",
|
|
"literature_dir": "Literature",
|
|
"control_dir": "LiteratureControl",
|
|
"base_dir": "05_Bases",
|
|
"skill_dir": ".opencode/skills",
|
|
},
|
|
indent=2,
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
# Create .env with PADDLEOCR_API_TOKEN
|
|
env_path = pf_dir / ".env"
|
|
env_path.write_text(
|
|
"PADDLEOCR_API_TOKEN=test_token\nPADDLEOCR_JOB_URL=https://example.com/api\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
# Copy OCR fixture
|
|
target_ocr = ocr_dir / "TSTONE001"
|
|
if OCR_FIXTURE.exists():
|
|
shutil.copytree(OCR_FIXTURE, target_ocr, dirs_exist_ok=True)
|
|
|
|
# Copy export fixture
|
|
if EXPORT_FIXTURE.exists():
|
|
shutil.copy2(EXPORT_FIXTURE, exports_dir / "骨科.json")
|
|
|
|
# Create library record for TSTONE001
|
|
domain_dir = records_dir / "骨科"
|
|
domain_dir.mkdir(parents=True, exist_ok=True)
|
|
record_path = domain_dir / "TSTONE001.md"
|
|
record_path.write_text(
|
|
"---\n"
|
|
'zotero_key: "TSTONE001"\n'
|
|
'domain: "骨科"\n'
|
|
'title: "Biomechanical Comparison of Suture Anchor Fixations in Rotator Cuff Repair"\n'
|
|
'year: "2024"\n'
|
|
'doi: "10.1016/j.jse.2024.01.001"\n'
|
|
'date: "2024-03-15"\n'
|
|
'collection_path: ""\n'
|
|
"has_pdf: true\n"
|
|
'pdf_path: "[[99_System/Zotero/storage/TSTONE001/TSTONE001.pdf]]"\n'
|
|
'fulltext_md_path: "[[99_System/PaperForge/ocr/TSTONE001/fulltext.md]]"\n'
|
|
"recommend_analyze: true\n"
|
|
"analyze: true\n"
|
|
"do_ocr: true\n"
|
|
'ocr_status: "done"\n'
|
|
'deep_reading_status: "pending"\n'
|
|
'analysis_note: ""\n'
|
|
"collection_group:\n"
|
|
' - "骨科"\n'
|
|
"collections:\n"
|
|
' - "骨科"\n'
|
|
"collection_tags:\n"
|
|
' - "骨科"\n'
|
|
'first_author: "John Smith"\n'
|
|
'journal: "Journal of Shoulder and Elbow Surgery"\n'
|
|
'impact_factor: ""\n'
|
|
"---\n\n"
|
|
"# Biomechanical Comparison of Suture Anchor Fixations in Rotator Cuff Repair\n\n"
|
|
"正式库控制记录。\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
# Create formal note for TSTONE001
|
|
note_path = (
|
|
literature_dir
|
|
/ "骨科"
|
|
/ "TSTONE001.md"
|
|
)
|
|
note_path.parent.mkdir(parents=True, exist_ok=True)
|
|
note_path.write_text(
|
|
"---\n"
|
|
'title: "Biomechanical Comparison of Suture Anchor Fixations in Rotator Cuff Repair"\n'
|
|
'year: "2024"\n'
|
|
'type: "journal"\n'
|
|
'journal: "Journal of Shoulder and Elbow Surgery"\n'
|
|
'impact_factor: "5.2"\n'
|
|
'category: "骨科"\n'
|
|
'zotero_key: "TSTONE001"\n'
|
|
'domain: "骨科"\n'
|
|
"analyze: true\n"
|
|
"do_ocr: true\n"
|
|
'ocr_status: "done"\n'
|
|
"tags:\n"
|
|
" - 文献阅读\n"
|
|
" - 骨科\n"
|
|
'keywords: ["biomechanics", "rotator cuff"]\n'
|
|
'pdf_link: "[[99_System/Zotero/storage/TSTONE001/TSTONE001.pdf]]"\n'
|
|
"---\n\n"
|
|
"# Biomechanical Comparison of Suture Anchor Fixations in Rotator Cuff Repair\n\n"
|
|
"## Abstract\n\n"
|
|
"This study compares the biomechanical properties...\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
# Create Zotero storage with mock PDF
|
|
# PDF goes at Zotero/KEY/filename.pdf so resolve_pdf_path can find it
|
|
zotero_dir = system_dir / "Zotero" / "TSTONE001"
|
|
zotero_dir.mkdir(parents=True, exist_ok=True)
|
|
(zotero_dir / "TSTONE001.pdf").write_text("mock pdf content", encoding="utf-8")
|
|
|
|
# Also create in storage/ subdirectory for legacy path resolution
|
|
storage_dir = system_dir / "Zotero" / "storage" / "TSTONE001"
|
|
storage_dir.mkdir(parents=True, exist_ok=True)
|
|
(storage_dir / "TSTONE001.pdf").write_text("mock pdf content", encoding="utf-8")
|
|
|
|
# Copy ld_deep.py to skill_dir (simulating deployment)
|
|
ld_deep_src = REPO_ROOT / "paperforge" / "skills" / "literature-qa" / "scripts" / "ld_deep.py"
|
|
if ld_deep_src.exists():
|
|
shutil.copy2(ld_deep_src, skill_dir / "ld_deep.py")
|
|
|
|
return vault
|
|
|
|
|
|
@pytest.fixture
|
|
def test_vault() -> Generator[Path, None, None]:
|
|
"""Pytest fixture providing a fresh test vault."""
|
|
vault = create_test_vault()
|
|
yield vault
|
|
# Cleanup
|
|
if FIXTURE_VAULT.exists():
|
|
shutil.rmtree(FIXTURE_VAULT)
|
|
|
|
|
|
@pytest.fixture
|
|
def test_vault_preserved() -> Generator[Path, None, None]:
|
|
"""Pytest fixture providing a test vault without automatic cleanup."""
|
|
vault = create_test_vault()
|
|
yield vault
|
|
|
|
@pytest.fixture(scope="session")
|
|
def e2e_fixture_dir() -> Path:
|
|
"""Return the path to E2E test fixture directories."""
|
|
return REPO_ROOT / "tests" / "fixtures"
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def synthetic_paper_paths(e2e_fixture_dir: Path) -> list[tuple[str, Path]]:
|
|
"""Return list of (paper_id, pdf_path) for synthetic E2E test papers."""
|
|
papers_dir = e2e_fixture_dir / "papers"
|
|
return [
|
|
("paper_a", papers_dir / "paper_a.pdf"),
|
|
("paper_b", papers_dir / "paper_b.pdf"),
|
|
("paper_c", papers_dir / "paper_c.pdf"),
|
|
]
|