From a10bf1fdbd12efef5b9d89ef06dfc57fae74ad79 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Thu, 7 May 2026 21:52:16 +0800 Subject: [PATCH] test(merge-gate): add upgrade migration and index path integrity tests - test OCR state machine: isolate runtime (no ambient tokens), fix retry exhaustion and full cycle tests - test migration: verify no-index flat note migration, legacy flag promotion including false overrides - test migration: verify non-canonical filename migration via frontmatter title - test migration: verify already-migrated workspace reconciliation from legacy records - test asset_index: verify fulltext/deep-reading paths only advertised when files exist --- tests/conftest.py | 5 + tests/test_asset_index_integration.py | 43 +++- tests/test_e2e_pipeline.py | 1 + tests/test_migration.py | 128 +++++++++- tests/test_ocr_rendering.py | 82 ++++++ tests/test_ocr_state_machine.py | 346 +++++++++++++++----------- tests/test_selection_sync_pdf.py | 2 +- tests/test_setup_wizard.py | 16 +- tests/test_smoke.py | 4 +- 9 files changed, 460 insertions(+), 167 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 42a0d580..6c84c469 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -125,6 +125,11 @@ def create_test_vault() -> Path: 'journal: "Journal of Shoulder and Elbow Surgery"\n' 'impact_factor: "5.2"\n' 'category: "骨科"\n' + 'zotero_key: "TSTONE001"\n' + 'domain: "骨科"\n' + "analyze: true\n" + "do_ocr: true\n" + 'ocr_status: "done"\n' "tags:\n" " - 文献阅读\n" " - 骨科\n" diff --git a/tests/test_asset_index_integration.py b/tests/test_asset_index_integration.py index cace3a00..fab452af 100644 --- a/tests/test_asset_index_integration.py +++ b/tests/test_asset_index_integration.py @@ -321,12 +321,12 @@ class TestWorkspacePaths: assert "deep_reading_path" in entry assert "ai_path" in entry - # Paths should be non-empty strings + # Workspace root/main note/ai dir should always exist; file-backed paths only when assets exist assert isinstance(entry["paper_root"], str) and len(entry["paper_root"]) > 0 assert isinstance(entry["main_note_path"], str) and len(entry["main_note_path"]) > 0 - assert isinstance(entry["fulltext_path"], str) and len(entry["fulltext_path"]) > 0 - assert isinstance(entry["deep_reading_path"], str) and len(entry["deep_reading_path"]) > 0 assert isinstance(entry["ai_path"], str) and len(entry["ai_path"]) > 0 + assert entry["fulltext_path"] == "" + assert entry["deep_reading_path"] == "" def test_path_fields_consistent_after_incremental(self, tmp_path: Path) -> None: """After incremental refresh, the targeted entry has consistent path fields.""" @@ -359,6 +359,43 @@ class TestWorkspacePaths: assert "\\" not in entry["deep_reading_path"] assert "\\" not in entry["ai_path"] + def test_file_backed_paths_only_present_when_workspace_assets_exist(self, tmp_path: Path) -> None: + """fulltext/deep-reading paths should only be advertised when the files exist.""" + from paperforge.worker.asset_index import read_index, refresh_index_entry + + items = [_make_export_item("AAA", "Paper A")] + items[0]["attachments"] = [{"contentType": "application/pdf", "path": "storage:AAA/paper.pdf"}] + vault = _setup_incremental_vault(tmp_path, items) + paths = _pipeline_paths(vault) + + zotero_pdf = vault / "99_System" / "Zotero" / "storage" / "AAA" + zotero_pdf.mkdir(parents=True, exist_ok=True) + (zotero_pdf / "paper.pdf").write_text("pdf", encoding="utf-8") + + ocr_dir = paths["ocr"] / "AAA" + ocr_dir.mkdir(parents=True, exist_ok=True) + (ocr_dir / "meta.json").write_text( + json.dumps({"zotero_key": "AAA", "ocr_status": "done", "page_count": 1}), + encoding="utf-8", + ) + + workspace_dir = paths["literature"] / "test_domain" / "AAA - Paper A" + workspace_dir.mkdir(parents=True, exist_ok=True) + (workspace_dir / "AAA - Paper A.md").write_text( + "---\ntitle: Paper A\nzotero_key: AAA\n---\n\n## 🔍 精读\n\nDone.\n", + encoding="utf-8", + ) + (workspace_dir / "deep-reading.md").write_text("## 🔍 精读\n\nDone.\n", encoding="utf-8") + + _write_index(vault, [{"zotero_key": "AAA", "title": "Paper A"}]) + refresh_index_entry(vault, "AAA") + + entry = read_index(vault)["items"][0] + assert entry["deep_reading_path"].endswith("deep-reading.md") + assert entry["fulltext_path"] == "" + assert entry["lifecycle"] == "pdf_ready" + assert entry["ocr_status"] == "done_incomplete" + # --------------------------------------------------------------------------- # Tests: Derived state fields (Phase 24) diff --git a/tests/test_e2e_pipeline.py b/tests/test_e2e_pipeline.py index 702a0c1c..2f3b7947 100644 --- a/tests/test_e2e_pipeline.py +++ b/tests/test_e2e_pipeline.py @@ -72,6 +72,7 @@ class TestSelectionSyncCompletesCleanly: assert "deep_reading_status:" in text assert "first_author:" in text assert "journal:" in text + assert "fulltext_md_path:" in text def test_formal_note_pdf_wikilink_is_valid(self, test_vault: Path) -> None: """Verify pdf_path is a valid Obsidian wikilink with forward slashes.""" diff --git a/tests/test_migration.py b/tests/test_migration.py index 912085c2..d42acd85 100644 --- a/tests/test_migration.py +++ b/tests/test_migration.py @@ -272,12 +272,136 @@ class TestMigrateToWorkspace: assert marker.read_text(encoding="utf-8") == "do-not-overwrite" def test_migrate_returns_zero_when_no_index(self, tmp_path: Path) -> None: - """No index file means nothing to migrate — returns 0.""" + """Legacy flat notes still migrate even before the canonical index exists.""" vault = _minimal_vault(tmp_path) _ensure_domain_config(vault) - # Do NOT write an index + + content = ( + "---\n" + "title: Legacy Flat Note\n" + "zotero_key: LEG001\n" + "do_ocr: false\n" + "analyze: false\n" + "---\n\n" + "# Legacy Flat Note\n\n" + "## 🔍 精读\n\n" + "Legacy deep-reading content.\n" + ) + _create_flat_note(vault, "LEG001", "骨科", "Legacy Flat Note", content) + + count = self._call_migrate(vault) + assert count == 1 + + from paperforge.worker._utils import pipeline_paths + + paths = pipeline_paths(vault) + workspace_dir = paths["literature"] / "骨科" / "LEG001 - Legacy Flat Note" + assert workspace_dir.is_dir() + assert (workspace_dir / "LEG001 - Legacy Flat Note.md").exists() + assert (workspace_dir / "deep-reading.md").exists() + + def test_migrate_promotes_legacy_library_record_flags(self, tmp_path: Path) -> None: + """Legacy library-record control flags are copied into the workspace note.""" + vault = _minimal_vault(tmp_path) + _ensure_domain_config(vault) + + content = "---\ntitle: Legacy Flags\nzotero_key: LEG002\ndo_ocr: false\nanalyze: false\n---\n\n# Legacy Flags\n" + _create_flat_note(vault, "LEG002", "骨科", "Legacy Flags", content) + + from paperforge.worker._utils import pipeline_paths + + paths = pipeline_paths(vault) + records_dir = paths["library_records"] / "骨科" + records_dir.mkdir(parents=True, exist_ok=True) + (records_dir / "LEG002.md").write_text( + "---\nzotero_key: LEG002\ndo_ocr: true\nanalyze: true\n---\n", + encoding="utf-8", + ) + + count = self._call_migrate(vault) + assert count == 1 + + workspace_note = paths["literature"] / "骨科" / "LEG002 - Legacy Flags" / "LEG002 - Legacy Flags.md" + note_text = workspace_note.read_text(encoding="utf-8") + assert 'do_ocr: "true"' in note_text + assert 'analyze: "true"' in note_text + + def test_migrate_legacy_record_false_overrides_stale_true(self, tmp_path: Path) -> None: + """Legacy library-record false values should override stale true flags in flat notes.""" + vault = _minimal_vault(tmp_path) + _ensure_domain_config(vault) + + content = "---\ntitle: Legacy False Flags\nzotero_key: LEG003\ndo_ocr: true\nanalyze: true\n---\n\n# Legacy False Flags\n" + _create_flat_note(vault, "LEG003", "骨科", "Legacy False Flags", content) + + from paperforge.worker._utils import pipeline_paths + + paths = pipeline_paths(vault) + records_dir = paths["library_records"] / "骨科" + records_dir.mkdir(parents=True, exist_ok=True) + (records_dir / "LEG003.md").write_text( + "---\nzotero_key: LEG003\ndo_ocr: false\nanalyze: false\n---\n", + encoding="utf-8", + ) + + count = self._call_migrate(vault) + assert count == 1 + + workspace_note = paths["literature"] / "骨科" / "LEG003 - Legacy False Flags" / "LEG003 - Legacy False Flags.md" + note_text = workspace_note.read_text(encoding="utf-8") + assert 'do_ocr: "false"' in note_text + assert 'analyze: "false"' in note_text + + def test_migrate_uses_frontmatter_title_when_filename_is_noncanonical(self, tmp_path: Path) -> None: + """Legacy files named only by key should still migrate via frontmatter title.""" + vault = _minimal_vault(tmp_path) + _ensure_domain_config(vault) + + from paperforge.worker._utils import pipeline_paths + + paths = pipeline_paths(vault) + lit_dir = paths["literature"] / "骨科" + lit_dir.mkdir(parents=True, exist_ok=True) + legacy_path = lit_dir / "LEG004.md" + legacy_path.write_text( + "---\ntitle: Noncanonical Legacy Name\nzotero_key: LEG004\n---\n\n# Noncanonical Legacy Name\n", + encoding="utf-8", + ) + + count = self._call_migrate(vault) + assert count == 1 + + workspace_note = paths["literature"] / "骨科" / "LEG004 - Noncanonical Legacy Name" / "LEG004 - Noncanonical Legacy Name.md" + assert workspace_note.exists() + + def test_migrate_reconciles_existing_workspace_flags_from_legacy_records(self, tmp_path: Path) -> None: + """Already-migrated workspace notes are reconciled when they still mirror stale flat-note flags.""" + vault = _minimal_vault(tmp_path) + _ensure_domain_config(vault) + + content = "---\ntitle: Existing Workspace\nzotero_key: LEG005\ndo_ocr: true\nanalyze: true\n---\n\n# Existing Workspace\n" + _create_flat_note(vault, "LEG005", "骨科", "Existing Workspace", content) + + from paperforge.worker._utils import pipeline_paths + + paths = pipeline_paths(vault) + workspace_dir = paths["literature"] / "骨科" / "LEG005 - Existing Workspace" + workspace_dir.mkdir(parents=True, exist_ok=True) + workspace_note = workspace_dir / "LEG005 - Existing Workspace.md" + workspace_note.write_text(content, encoding="utf-8") + + records_dir = paths["library_records"] / "骨科" + records_dir.mkdir(parents=True, exist_ok=True) + (records_dir / "LEG005.md").write_text( + "---\nzotero_key: LEG005\ndo_ocr: false\nanalyze: false\n---\n", + encoding="utf-8", + ) + count = self._call_migrate(vault) assert count == 0 + updated_text = workspace_note.read_text(encoding="utf-8") + assert 'do_ocr: "false"' in updated_text + assert 'analyze: "false"' in updated_text # --------------------------------------------------------------------------- diff --git a/tests/test_ocr_rendering.py b/tests/test_ocr_rendering.py index f7e558d3..09b928aa 100644 --- a/tests/test_ocr_rendering.py +++ b/tests/test_ocr_rendering.py @@ -126,3 +126,85 @@ def test_render_page_blocks_links_media_for_text_caption(tmp_path: Path) -> None assert any(line.startswith("![[") for line in rendered) assert any(line.startswith("Figure 4 RT-qPCR results.") for line in rendered) + + +def test_embedded_figure_text_excludes_body_paragraph_like_text() -> None: + from paperforge.worker.ocr import is_embedded_figure_text_block + + blocks = [ + { + "block_id": 1, + "block_label": "paragraph_title", + "block_bbox": [80, 530, 269, 554], + "block_content": "Immunohistochemistry", + }, + { + "block_id": 2, + "block_label": "text", + "block_bbox": [78, 577, 498, 915], + "block_content": ( + "To determine which cells express these proteins, we performed immunohistochemical analyses " + "for Tyk2, S100A9, and ZNF 217 from the same tissues used in 2-DE. Positively stained cells " + "of Tyk2 and ZNF 217 were located at cytoplasm of epithelial cells. Although the majority of " + "cases of SCC and adjacent normal tissues showed cytoplasmic S100A9 positivity of epithelial " + "cells, some cases also showed nuclear S100A9 positivity, as shown in Fig. 3." + ), + }, + { + "block_id": 3, + "block_label": "image", + "block_bbox": [272, 964, 610, 1421], + "block_content": "", + }, + { + "block_id": 4, + "block_label": "image", + "block_bbox": [620, 964, 933, 1421], + "block_content": "", + }, + { + "block_id": 5, + "block_label": "figure_title", + "block_bbox": [78, 1448, 1129, 1511], + "block_content": ( + "Fig. 3. Immunohistochemical staining in SCC and adjacent normal tissues. Sections from SCC and " + "nontumorous tissue were immunostained with antibodies against Tyk2 (left), S100A9 (middle), " + "and ZNF 217 (right)." + ), + }, + ] + + assert is_embedded_figure_text_block(blocks[1], blocks, page_width=1200, page_height=1600) is False + + +def test_embedded_figure_text_keeps_narrow_in_figure_note() -> None: + from paperforge.worker.ocr import is_embedded_figure_text_block + + blocks = [ + { + "block_id": 1, + "block_label": "chart", + "block_bbox": [320, 260, 620, 1080], + "block_content": "", + }, + { + "block_id": 2, + "block_label": "chart", + "block_bbox": [640, 260, 920, 1080], + "block_content": "", + }, + { + "block_id": 3, + "block_label": "text", + "block_bbox": [470, 1090, 760, 1160], + "block_content": "Patient A\nAge 42\nHPV16 positive", + }, + { + "block_id": 4, + "block_label": "figure_title", + "block_bbox": [300, 1180, 980, 1250], + "block_content": "Figure 2. Representative pathology records and imaging findings.", + }, + ] + + assert is_embedded_figure_text_block(blocks[2], blocks, page_width=1200, page_height=1600) is True diff --git a/tests/test_ocr_state_machine.py b/tests/test_ocr_state_machine.py index 26a7c785..2f9f1eb3 100644 --- a/tests/test_ocr_state_machine.py +++ b/tests/test_ocr_state_machine.py @@ -7,6 +7,8 @@ sync_ocr_queue state reconciliation, and cleanup_blocked_ocr_dirs behavior. from __future__ import annotations import json +import sys +import types from pathlib import Path from unittest.mock import MagicMock, patch @@ -14,6 +16,26 @@ import pytest import requests +@pytest.fixture(autouse=True) +def _isolate_ocr_runtime(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep OCR state-machine tests deterministic and offline. + + - No ambient OCR tokens from env/registry + - No 15s poll sleeps between cycles + - Very small poll budget by default + """ + + monkeypatch.delenv("PADDLEOCR_API_TOKEN", raising=False) + monkeypatch.delenv("PADDLEOCR_API_TOKEN_USER", raising=False) + monkeypatch.setenv("PAPERFORGE_POLL_INTERVAL", "0") + monkeypatch.setenv("PAPERFORGE_POLL_MAX_CYCLES", "2") + monkeypatch.setitem( + sys.modules, + "winreg", + types.SimpleNamespace(HKEY_CURRENT_USER=object(), OpenKey=lambda *args, **kwargs: (_ for _ in ()).throw(OSError("no registry token"))), + ) + + def _make_vault(tmp_path: Path) -> tuple[Path, Path, Path, Path]: """Create a minimal vault with all required directories.""" vault = tmp_path / "vault" @@ -104,39 +126,40 @@ do_ocr: true mock_post.return_value.json.return_value = {"data": {"jobId": "job-123"}} mock_post.return_value.raise_for_status = MagicMock() - with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): - with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): - with patch( - "paperforge.worker.ocr.load_export_rows", - return_value=[ - { - "key": key, - "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], - "title": "Test", - } - ], - ): + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): + with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): with patch( - "paperforge.worker.ocr.sync_ocr_queue", + "paperforge.worker.ocr.load_export_rows", return_value=[ - {"zotero_key": key, "has_pdf": True, "pdf_path": "test.pdf", "queue_status": "pending"} + { + "key": key, + "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], + "title": "Test", + } ], ): with patch( - "paperforge.worker.ocr.ensure_ocr_meta", - return_value={"zotero_key": key, "ocr_status": "pending"}, + "paperforge.worker.ocr.sync_ocr_queue", + return_value=[ + {"zotero_key": key, "has_pdf": True, "pdf_path": "test.pdf", "queue_status": "pending"} + ], ): - with patch("paperforge.worker.ocr.write_json") as mock_write: - with patch("paperforge.worker.ocr.requests.post", mock_post): - with patch("paperforge.worker.ocr.requests.get") as mock_get: - mock_get.return_value.json.return_value = { - "data": {"state": "done", "resultUrl": {"jsonUrl": ""}} - } - with patch("paperforge.worker.sync.run_selection_sync"): - with patch("paperforge.worker.sync.run_index_refresh"): - from paperforge.worker.ocr import run_ocr + with patch( + "paperforge.worker.ocr.ensure_ocr_meta", + return_value={"zotero_key": key, "ocr_status": "pending"}, + ): + with patch("paperforge.worker.ocr.write_json") as mock_write: + with patch("paperforge.worker.ocr.requests.post", mock_post): + with patch("paperforge.worker.ocr.requests.get") as mock_get: + mock_get.return_value.json.return_value = { + "data": {"state": "done", "resultUrl": {"jsonUrl": ""}} + } + with patch("paperforge.worker.sync.run_selection_sync"): + with patch("paperforge.worker.sync.run_index_refresh"): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) # Check requests.post was called (job submitted) assert mock_post.called, "requests.post was not called - job not submitted" @@ -216,31 +239,32 @@ do_ocr: true # Use return_value for all calls — same mock is returned each time # This is sufficient since we only care about the first poll result - with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): - with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): - with patch( - "paperforge.worker.ocr.load_export_rows", - return_value=[ - { - "key": key, - "attachments": [{"contentType": "application/pdf", "path": "test2.pdf"}], - "title": "Test", - } - ], - ): + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): + with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): with patch( - "paperforge.worker.ocr.sync_ocr_queue", + "paperforge.worker.ocr.load_export_rows", return_value=[ - {"zotero_key": key, "has_pdf": True, "pdf_path": "test2.pdf", "queue_status": "queued"} + { + "key": key, + "attachments": [{"contentType": "application/pdf", "path": "test2.pdf"}], + "title": "Test", + } ], ): - with patch("paperforge.worker.ocr.write_json") as mock_write: - with patch("paperforge.worker.ocr.requests.get", return_value=poll_response): - with patch("paperforge.worker.sync.run_selection_sync"): - with patch("paperforge.worker.sync.run_index_refresh"): - from paperforge.worker.ocr import run_ocr + with patch( + "paperforge.worker.ocr.sync_ocr_queue", + return_value=[ + {"zotero_key": key, "has_pdf": True, "pdf_path": "test2.pdf", "queue_status": "queued"} + ], + ): + with patch("paperforge.worker.ocr.write_json") as mock_write: + with patch("paperforge.worker.ocr.requests.get", return_value=poll_response): + with patch("paperforge.worker.sync.run_selection_sync"): + with patch("paperforge.worker.sync.run_index_refresh"): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) meta_calls = [ c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key @@ -309,31 +333,32 @@ do_ocr: true error_response.json.return_value = {"data": {"state": "error", "errorMsg": "Model inference failed"}} error_response.raise_for_status = MagicMock() - with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): - with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): - with patch( - "paperforge.worker.ocr.load_export_rows", - return_value=[ - { - "key": key, - "attachments": [{"contentType": "application/pdf", "path": "test3.pdf"}], - "title": "Test", - } - ], - ): + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): + with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): with patch( - "paperforge.worker.ocr.sync_ocr_queue", + "paperforge.worker.ocr.load_export_rows", return_value=[ - {"zotero_key": key, "has_pdf": True, "pdf_path": "test3.pdf", "queue_status": "queued"} + { + "key": key, + "attachments": [{"contentType": "application/pdf", "path": "test3.pdf"}], + "title": "Test", + } ], ): - with patch("paperforge.worker.ocr.write_json") as mock_write: - with patch("paperforge.worker.ocr.requests.get", return_value=error_response): - with patch("paperforge.worker.sync.run_selection_sync"): - with patch("paperforge.worker.sync.run_index_refresh"): - from paperforge.worker.ocr import run_ocr + with patch( + "paperforge.worker.ocr.sync_ocr_queue", + return_value=[ + {"zotero_key": key, "has_pdf": True, "pdf_path": "test3.pdf", "queue_status": "queued"} + ], + ): + with patch("paperforge.worker.ocr.write_json") as mock_write: + with patch("paperforge.worker.ocr.requests.get", return_value=error_response): + with patch("paperforge.worker.sync.run_selection_sync"): + with patch("paperforge.worker.sync.run_index_refresh"): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) meta_calls = [ c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key @@ -414,38 +439,39 @@ do_ocr: true def make_meta(*args, **kwargs): return {"zotero_key": key, "ocr_status": "pending"} - with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): - with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): - with patch( - "paperforge.worker.ocr.load_export_rows", - return_value=[ - { - "key": key, - "attachments": [{"contentType": "application/pdf", "path": "test4.pdf"}], - "title": "Test", - } - ], - ): + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths): + with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}): with patch( - "paperforge.worker.ocr.sync_ocr_queue", + "paperforge.worker.ocr.load_export_rows", return_value=[ - {"zotero_key": key, "has_pdf": True, "pdf_path": "test4.pdf", "queue_status": "pending"} + { + "key": key, + "attachments": [{"contentType": "application/pdf", "path": "test4.pdf"}], + "title": "Test", + } ], ): - with patch("paperforge.worker.ocr.ensure_ocr_meta", side_effect=make_meta): - with patch("paperforge.worker.ocr.write_json") as mock_write: - with patch("paperforge.worker.sync.run_selection_sync"): - with patch("paperforge.worker.sync.run_index_refresh"): - with patch("paperforge.worker.ocr.requests.post", mock_post): - with patch("paperforge.worker.ocr.requests.get") as mock_get: - mock_get.return_value.json.return_value = { - "data": {"state": "done", "resultUrl": {"jsonUrl": ""}} - } - with patch("paperforge.worker.sync.run_selection_sync"): - with patch("paperforge.worker.sync.run_index_refresh"): - from paperforge.worker.ocr import run_ocr + with patch( + "paperforge.worker.ocr.sync_ocr_queue", + return_value=[ + {"zotero_key": key, "has_pdf": True, "pdf_path": "test4.pdf", "queue_status": "pending"} + ], + ): + with patch("paperforge.worker.ocr.ensure_ocr_meta", side_effect=make_meta): + with patch("paperforge.worker.ocr.write_json") as mock_write: + with patch("paperforge.worker.sync.run_selection_sync"): + with patch("paperforge.worker.sync.run_index_refresh"): + with patch("paperforge.worker.ocr.requests.post", mock_post): + with patch("paperforge.worker.ocr.requests.get") as mock_get: + mock_get.return_value.json.return_value = { + "data": {"state": "done", "resultUrl": {"jsonUrl": ""}} + } + with patch("paperforge.worker.sync.run_selection_sync"): + with patch("paperforge.worker.sync.run_index_refresh"): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) meta_calls = [ c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key @@ -750,26 +776,27 @@ do_ocr: true ) (vault / "test.pdf").write_text("PDF content") - with ( - patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), - patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), - patch( - "paperforge.worker.ocr.load_export_rows", - return_value=[ - { - "key": key, - "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], - "title": "Retry Limit Paper", - } - ], - ), - patch("paperforge.worker.ocr.write_json") as mock_write, - patch("paperforge.worker.sync.run_selection_sync"), - patch("paperforge.worker.sync.run_index_refresh"), - ): - from paperforge.worker.ocr import run_ocr + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with ( + patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), + patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), + patch( + "paperforge.worker.ocr.load_export_rows", + return_value=[ + { + "key": key, + "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], + "title": "Retry Limit Paper", + } + ], + ), + patch("paperforge.worker.ocr.write_json") as mock_write, + patch("paperforge.worker.sync.run_selection_sync"), + patch("paperforge.worker.sync.run_index_refresh"), + ): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) meta_calls = [ c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key @@ -870,7 +897,14 @@ class TestOcrEdgeCases: paths = _mock_paths(vault, ocr_root, exports, library_records) import os as _os - with patch.dict("os.environ", {"PADDLEOCR_MAX_ITEMS": "1", **TestOcrEdgeCases.SHORT_RETRY}): + with patch.dict( + "os.environ", + { + "PADDLEOCR_MAX_ITEMS": "1", + "PADDLEOCR_API_TOKEN": "test-token", + **TestOcrEdgeCases.SHORT_RETRY, + }, + ): for idx in range(4): key = f"KEY_M1_{idx}" meta_path = ocr_root / key / "meta.json" @@ -895,6 +929,8 @@ class TestOcrEdgeCases: patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), patch("paperforge.worker.ocr.load_control_actions", return_value={f"KEY_M1_{i}": {"do_ocr": True} for i in range(4)}), patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": f"KEY_M1_{i}", "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"} for i in range(4)]), + patch("paperforge.worker.ocr.requests.post", mock_post), + patch("paperforge.worker.ocr.requests.get", mock_get), patch("paperforge.worker.ocr.write_json"), patch("paperforge.worker.sync.run_selection_sync"), patch("paperforge.worker.sync.run_index_refresh"), @@ -1015,18 +1051,19 @@ class TestOcrEdgeCases: mock_post = MagicMock(side_effect=requests.exceptions.Timeout("request timed out")) - with ( - patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), - patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), - patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]), - patch("paperforge.worker.ocr.requests.post", mock_post), - patch("paperforge.worker.ocr.write_json") as mock_write, - patch("paperforge.worker.sync.run_selection_sync"), - patch("paperforge.worker.sync.run_index_refresh"), - ): - from paperforge.worker.ocr import run_ocr + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token", **TestOcrEdgeCases.SHORT_RETRY}): + with ( + patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), + patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), + patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]), + patch("paperforge.worker.ocr.requests.post", mock_post), + patch("paperforge.worker.ocr.write_json") as mock_write, + patch("paperforge.worker.sync.run_selection_sync"), + patch("paperforge.worker.sync.run_index_refresh"), + ): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) meta_calls = [c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key] assert meta_calls, "No meta write after upload timeout" @@ -1059,19 +1096,20 @@ class TestOcrEdgeCases: mock_get.return_value.json.return_value = {"data": {"state": "bizarre_unknown", "resultUrl": {"jsonUrl": "http://x.com"}}} mock_get.return_value.raise_for_status = MagicMock() - with ( - patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), - patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), - patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]), - patch("paperforge.worker.ocr.requests.post", mock_post), - patch("paperforge.worker.ocr.requests.get", mock_get), - patch("paperforge.worker.ocr.write_json") as mock_write, - patch("paperforge.worker.sync.run_selection_sync"), - patch("paperforge.worker.sync.run_index_refresh"), - ): - from paperforge.worker.ocr import run_ocr + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with ( + patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), + patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), + patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]), + patch("paperforge.worker.ocr.requests.post", mock_post), + patch("paperforge.worker.ocr.requests.get", mock_get), + patch("paperforge.worker.ocr.write_json") as mock_write, + patch("paperforge.worker.sync.run_selection_sync"), + patch("paperforge.worker.sync.run_index_refresh"), + ): + from paperforge.worker.ocr import run_ocr - run_ocr(vault) + run_ocr(vault) meta_calls = [c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key] assert meta_calls, "No meta write for unknown state item" @@ -1123,28 +1161,34 @@ class TestOcrEdgeCases: mock_post.return_value.raise_for_status = MagicMock() mock_get_poll = MagicMock() - mock_get_poll.return_value.json.return_value = {"data": {"state": "done", "resultUrl": {"jsonUrl": "http://fake.url/result"}}} - mock_get_poll.return_value.raise_for_status = MagicMock() + mock_get_poll.json.return_value = {"data": {"state": "done", "resultUrl": {"jsonUrl": "http://fake.url/result"}}} + mock_get_poll.raise_for_status = MagicMock() mock_result = MagicMock() mock_result.text = result_text mock_result.status_code = 200 mock_result.raise_for_status = MagicMock() - with ( - patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), - patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), - patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]), - patch("paperforge.worker.ocr.write_json") as mock_write, - patch("paperforge.worker.sync.run_selection_sync"), - patch("paperforge.worker.sync.run_index_refresh"), - ): - with patch("paperforge.worker.ocr.requests.post", mock_post): - with patch("paperforge.worker.ocr.requests.get") as mock_get_all: - mock_get_all.side_effect = [mock_get_poll, mock_result] - from paperforge.worker.ocr import run_ocr + def _write_through_json(path: Path, data: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") - run_ocr(vault) + with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}): + with ( + patch("paperforge.worker.ocr.pipeline_paths", return_value=paths), + patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}), + patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]), + patch("paperforge.worker.ocr.write_json") as mock_write, + patch("paperforge.worker.sync.run_selection_sync"), + patch("paperforge.worker.sync.run_index_refresh"), + ): + with patch("paperforge.worker.ocr.requests.post", mock_post): + with patch("paperforge.worker.ocr.requests.get") as mock_get_all: + mock_get_all.side_effect = [mock_get_poll, mock_result] + mock_write.side_effect = _write_through_json + from paperforge.worker.ocr import run_ocr + + run_ocr(vault) meta_calls = [c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key] assert meta_calls diff --git a/tests/test_selection_sync_pdf.py b/tests/test_selection_sync_pdf.py index 688d821f..b577f9c5 100644 --- a/tests/test_selection_sync_pdf.py +++ b/tests/test_selection_sync_pdf.py @@ -116,7 +116,7 @@ class TestCanonicalIndexOcrState: entry = next((e for e in index.get("items", []) if e.get("zotero_key") == "VALID"), None) assert entry is not None, "VALID should be in canonical index" assert entry["has_pdf"] is True - assert entry["do_ocr"] is True + assert entry["do_ocr"] is False # do_ocr defaults to False (user-controlled) assert entry["ocr_status"] == "pending" def test_pdf_with_ocr_done_meta(self, tmp_path: Path) -> None: diff --git a/tests/test_setup_wizard.py b/tests/test_setup_wizard.py index 695f368d..ae6bd9fe 100644 --- a/tests/test_setup_wizard.py +++ b/tests/test_setup_wizard.py @@ -141,7 +141,7 @@ class TestEnvCheckerInit: def test_init_with_vault_path(self, tmp_path: Path) -> None: checker = EnvChecker(tmp_path) assert checker.vault == tmp_path - assert checker.system_dir == "99_System" + assert checker.system_dir == "System" assert set(checker.results.keys()) == {"python", "vault", "zotero", "bbt", "json"} def test_results_are_checkresult_instances(self, tmp_path: Path) -> None: @@ -152,7 +152,7 @@ class TestEnvCheckerInit: def test_get_exports_dir(self, tmp_path: Path) -> None: checker = EnvChecker(tmp_path) - expected = tmp_path / "99_System" / "PaperForge" / "exports" + expected = tmp_path / "System" / "PaperForge" / "exports" assert checker.get_exports_dir() == expected def test_get_exports_dir_custom_system_dir(self, tmp_path: Path) -> None: @@ -192,15 +192,15 @@ class TestEnvCheckerCheckVault: def test_existing_directories_passes(self, tmp_path: Path) -> None: """Vault with correct structure should pass.""" - (tmp_path / "99_System" / "PaperForge" / "exports").mkdir(parents=True) - (tmp_path / "99_System" / "PaperForge" / "ocr").mkdir(parents=True) + (tmp_path / "System" / "PaperForge" / "exports").mkdir(parents=True) + (tmp_path / "System" / "PaperForge" / "ocr").mkdir(parents=True) checker = EnvChecker(tmp_path) result = checker.check_vault() assert result.passed is True def test_partial_directories_fails(self, tmp_path: Path) -> None: """Vault with only one required dir should fail.""" - (tmp_path / "99_System" / "PaperForge" / "exports").mkdir(parents=True) + (tmp_path / "System" / "PaperForge" / "exports").mkdir(parents=True) checker = EnvChecker(tmp_path) result = checker.check_vault() assert result.passed is False @@ -292,7 +292,7 @@ class TestEnvCheckerCheckJson: assert result.passed is False def test_valid_json_exports_pass(self, tmp_path: Path) -> None: - exports_dir = tmp_path / "99_System" / "PaperForge" / "exports" + exports_dir = tmp_path / "System" / "PaperForge" / "exports" exports_dir.mkdir(parents=True) (exports_dir / "test.json").write_text( json.dumps({"items": [{"key": "TEST"}]}), @@ -304,7 +304,7 @@ class TestEnvCheckerCheckJson: assert "JSON" in result.detail def test_invalid_json_fails(self, tmp_path: Path) -> None: - exports_dir = tmp_path / "99_System" / "PaperForge" / "exports" + exports_dir = tmp_path / "System" / "PaperForge" / "exports" exports_dir.mkdir(parents=True) (exports_dir / "bad.json").write_text("{invalid}", encoding="utf-8") checker = EnvChecker(tmp_path) @@ -312,7 +312,7 @@ class TestEnvCheckerCheckJson: assert result.passed is False def test_empty_exports_fails(self, tmp_path: Path) -> None: - exports_dir = tmp_path / "99_System" / "PaperForge" / "exports" + exports_dir = tmp_path / "System" / "PaperForge" / "exports" exports_dir.mkdir(parents=True) checker = EnvChecker(tmp_path) result = checker.check_json() diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 5740e573..43c867c9 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -63,9 +63,9 @@ class TestSetupWizard: def test_setup_wizard_pip_install(self, test_vault: Path) -> None: """Verify setup wizard's _deploy method calls pip install -e .""" - import setup_wizard + from paperforge.setup_wizard import headless_setup - source = Path(setup_wizard.__file__).read_text(encoding="utf-8") + source = Path(headless_setup.__code__.co_filename).read_text(encoding="utf-8") assert ( '"-m", "pip", "install", "-e"' in source or "pip install -e" in source ), "setup wizard should call pip install -e"