From ed869dffaa98fcaeb3288adcd9617429e24bd624 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Wed, 20 May 2026 15:47:13 +0800 Subject: [PATCH] test: add fixture-driven annotation import coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fixtures/zotero/build_test_db.py generates minimal Zotero SQLite with highlight/underline/note annotations and tags - tests/integration/test_annotation_import_workflow.py end-to-end probe→import→verify pipeline using a real SQLite fixture (generated on demand, not checked in) - .gitignore excludes generated *.sqlite from fixtures/zotero --- .gitignore | 3 + fixtures/zotero/build_test_db.py | 98 +++++++++++ .../test_annotation_import_workflow.py | 159 ++++++++++++++++++ 3 files changed, 260 insertions(+) create mode 100644 fixtures/zotero/build_test_db.py create mode 100644 tests/integration/test_annotation_import_workflow.py diff --git a/.gitignore b/.gitignore index 02a5bc7c..71ff086a 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ Thumbs.db .vite/ paperforge/plugin/node_modules/ + +# Generated SQLite fixtures +fixtures/zotero/*.sqlite \ No newline at end of file diff --git a/fixtures/zotero/build_test_db.py b/fixtures/zotero/build_test_db.py new file mode 100644 index 00000000..2d0093cf --- /dev/null +++ b/fixtures/zotero/build_test_db.py @@ -0,0 +1,98 @@ +"""Build a minimal Zotero SQLite fixture for annotation import testing. + +Usage: python fixtures/zotero/build_test_db.py [--output PATH] + +Default output: fixtures/zotero/test_annotations.sqlite (alongside this script) +""" + +from __future__ import annotations + +import argparse +import sqlite3 +from pathlib import Path + + +def build_test_db(path: Path) -> None: + conn = sqlite3.connect(str(path)) + conn.execute("PRAGMA journal_mode=OFF") + + conn.executescript(""" + CREATE TABLE libraries ( + libraryID INTEGER PRIMARY KEY, type TEXT, editable INT, + filesEditable INT, version INT, storageVersion INT, + lastSync INT, archived INT, isAdmin INT + ); + INSERT INTO libraries VALUES (1, 'user', 1, 1, 1, 0, 0, 0, 0); + + CREATE TABLE items ( + itemID INTEGER PRIMARY KEY, itemTypeID INT NOT NULL, + dateAdded TEXT, dateModified TEXT, clientDateModified TEXT, + libraryID INT NOT NULL, key TEXT NOT NULL, + version INT DEFAULT 0, synced INT DEFAULT 0 + ); + -- Top-level paper + INSERT INTO items VALUES (1, 1, '2025-01-01', '2025-01-02', '2025-01-01', 1, 'PAPER001', 5, 1); + -- PDF attachment + INSERT INTO items VALUES (2, 2, '2025-01-01', '2025-01-02', '2025-01-01', 1, 'ATTACH01', 5, 1); + -- Annotations + INSERT INTO items VALUES (3, 3, '2025-01-02', '2025-01-03', '2025-01-02', 1, 'ANNOT001', 3, 1); + INSERT INTO items VALUES (4, 3, '2025-01-02', '2025-01-03', '2025-01-02', 1, 'ANNOT002', 2, 1); + INSERT INTO items VALUES (5, 3, '2025-01-03', '2025-01-04', '2025-01-03', 1, 'ANNOT003', 1, 1); + + CREATE TABLE itemTypes (itemTypeID INTEGER PRIMARY KEY, typeName TEXT); + INSERT INTO itemTypes VALUES (1, 'journalArticle'); + INSERT INTO itemTypes VALUES (2, 'attachment'); + INSERT INTO itemTypes VALUES (3, 'annotation'); + + CREATE TABLE itemAttachments ( + itemID INTEGER PRIMARY KEY, parentItemID INT, + linkMode INT, contentType TEXT, path TEXT + ); + INSERT INTO itemAttachments VALUES (2, 1, 0, 'application/pdf', 'storage:ATTACH01/paper.pdf'); + + CREATE TABLE itemAnnotations ( + itemID INTEGER PRIMARY KEY, parentItemID INT NOT NULL, + type INTEGER NOT NULL, authorName TEXT, text TEXT, comment TEXT, + color TEXT, pageLabel TEXT, sortIndex TEXT NOT NULL, + position TEXT NOT NULL, isExternal INT NOT NULL + ); + -- highlight + INSERT INTO itemAnnotations VALUES (3, 2, 1, '', + 'Deep learning methods are effective for image segmentation.', + 'Important finding', '#ffd400', '3', + '00002|000000|00000', + '{"pageIndex":2,"rects":[[72,520,540,536],[72,504,540,520]]}', 0); + -- underline + INSERT INTO itemAnnotations VALUES (4, 2, 5, '', + 'The primary limitation is the need for large annotated datasets.', + 'Related work section', '#ff6666', '12', + '00011|000000|00000', + '{"pageIndex":11,"rects":[[72,480,540,496]]}', 0); + -- note + INSERT INTO itemAnnotations VALUES (5, 2, 2, '', '', + '

Key figure explaining U-Net architecture.

', '#2ea8e5', '7', + '00006|000000|00000', + '{"pageIndex":6,"rects":[[420,360,540,400]]}', 0); + + CREATE TABLE tags (tagID INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE); + INSERT INTO tags VALUES (1, 'deep_learning'); + INSERT INTO tags VALUES (2, 'methods'); + + CREATE TABLE itemTags ( + itemID INT NOT NULL, tagID INT NOT NULL, type INT NOT NULL, + PRIMARY KEY (itemID, tagID) + ); + INSERT INTO itemTags VALUES (3, 1, 0); + INSERT INTO itemTags VALUES (3, 2, 0); + """) + conn.commit() + conn.close() + print(f"Built test fixture at {path}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args() + output = args.output or (Path(__file__).resolve().parent / "test_annotations.sqlite") + build_test_db(output) diff --git a/tests/integration/test_annotation_import_workflow.py b/tests/integration/test_annotation_import_workflow.py new file mode 100644 index 00000000..8c7a79dc --- /dev/null +++ b/tests/integration/test_annotation_import_workflow.py @@ -0,0 +1,159 @@ +"""Integration test: fixture-driven Zotero annotation import workflow. + +Requires the fixture at fixtures/zotero/test_annotations.sqlite, which can +be generated by running: + python fixtures/zotero/build_test_db.py +""" + +from __future__ import annotations + +import shutil +import sqlite3 +import tempfile +from pathlib import Path + +import pytest + +FIXTURE_DIR = Path(__file__).resolve().parent.parent.parent / "fixtures" / "zotero" +FIXTURE_PATH = FIXTURE_DIR / "test_annotations.sqlite" + + +@pytest.fixture(scope="session") +def zotero_fixture() -> Path: + """Ensure the fixture exists (generate if needed).""" + if not FIXTURE_PATH.exists(): + from fixtures.zotero.build_test_db import build_test_db + + FIXTURE_DIR.mkdir(parents=True, exist_ok=True) + build_test_db(FIXTURE_PATH) + return FIXTURE_PATH + + +@pytest.fixture +def zotero_copy(zotero_fixture: Path) -> Path: + """Copy the fixture to a temp location for each test.""" + tmp = Path(tempfile.mkdtemp()) / "zotero_copy.sqlite" + shutil.copy2(str(zotero_fixture), str(tmp)) + yield tmp + shutil.rmtree(tmp.parent, ignore_errors=True) + + +@pytest.fixture +def ann_db(tmp_path: Path): + """Provide a fresh annotation database with schema applied.""" + from paperforge.annotation.db import get_annotations_db_path, get_annotations_connection + from paperforge.annotation.schema import ensure_schema + + db_path = get_annotations_db_path(tmp_path) + conn = get_annotations_connection(db_path, read_only=False) + ensure_schema(conn) + yield conn + conn.close() + + +class TestZoteroImportWorkflow: + """End-to-end import from a real Zotero-style SQLite fixture.""" + + def test_probe_reads_all_annotation_types(self, zotero_copy): + """Verify probe can read highlight, underline, and note from fixture.""" + from paperforge.annotation.probe import open_readonly, fetch_annotations + + conn = open_readonly(zotero_copy) + try: + anns = fetch_annotations(conn, limit=10) + finally: + conn.close() + + assert len(anns) == 3 + types = {a["annotationType"] for a in anns} + assert types == {"highlight", "underline", "note"} + + def test_full_import_pipeline(self, zotero_copy, ann_db): + """Probe → import → verify.""" + from paperforge.annotation.probe import open_readonly, fetch_annotations + from paperforge.annotation.importer import run_import + from paperforge.annotation.service import list_annotations + + # Phase 1: probe + probe_conn = open_readonly(zotero_copy) + try: + anns = fetch_annotations(probe_conn, limit=10) + finally: + probe_conn.close() + + assert len(anns) == 3 + + # Phase 2: import + result = run_import(ann_db, anns, source="zotero_db") + assert result["imported"] == 3 + assert result["updated"] == 0 + assert result["deleted"] == 0 + + # Phase 3: verify + stored = list_annotations(ann_db, paper_id="PAPER001") + assert len(stored) == 3 + + highlight = next(a for a in stored if a["type"] == "highlight") + assert highlight["selected_text"].startswith("Deep learning") + assert highlight["color"] == "#ffd400" + assert highlight["source"] == "zotero_db" + assert highlight["sync_state"] == "zotero_synced" + assert highlight["is_readonly"] == 1 + + underline = next(a for a in stored if a["type"] == "underline") + assert underline["selected_text"].startswith("The primary limitation") + assert underline["color"] == "#ff6666" + + note = next(a for a in stored if a["type"] == "note") + assert "U-Net" in note["comment"] + + def test_import_is_idempotent(self, zotero_copy, ann_db): + """Re-importing same data should not create duplicates.""" + from paperforge.annotation.probe import open_readonly, fetch_annotations + from paperforge.annotation.importer import run_import + + probe_conn = open_readonly(zotero_copy) + try: + anns = fetch_annotations(probe_conn, limit=10) + finally: + probe_conn.close() + + r1 = run_import(ann_db, anns, source="zotero_db") + r2 = run_import(ann_db, anns, source="zotero_db") + + assert r1["imported"] == 3 + assert r2["imported"] == 0 + assert r2["updated"] == 0 + + count = ann_db.execute( + "SELECT COUNT(*) as cnt FROM annotations WHERE deleted_at IS NULL" + ).fetchone()["cnt"] + assert count == 3 + + def test_reimport_updated_annotation(self, zotero_copy, ann_db): + """Import, modify fixture version, reimport — verify update.""" + from paperforge.annotation.probe import open_readonly, fetch_annotations + from paperforge.annotation.importer import run_import + + probe_conn = open_readonly(zotero_copy) + try: + anns = fetch_annotations(probe_conn, limit=10) + finally: + probe_conn.close() + + run_import(ann_db, anns, source="zotero_db") + + # Simulate updated annotation by changing the fixture + zotero_copy2 = zotero_copy # not modifying, just changing the dict + anns[0]["comment"] = "Updated comment" + anns[0]["version"] = 4 + + result = run_import(ann_db, anns, source="zotero_db") + assert result["updated"] >= 1 + + stored = ann_db.execute( + "SELECT comment, source_version FROM annotations WHERE zotero_key = ?", + (anns[0]["annotationKey"],), + ).fetchone() + assert stored["comment"] == "Updated comment" + assert stored["source_version"] == 4