mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
test(merge-gate): add upgrade migration and index path integrity tests
- test OCR state machine: isolate runtime (no ambient tokens), fix retry exhaustion and full cycle tests - test migration: verify no-index flat note migration, legacy flag promotion including false overrides - test migration: verify non-canonical filename migration via frontmatter title - test migration: verify already-migrated workspace reconciliation from legacy records - test asset_index: verify fulltext/deep-reading paths only advertised when files exist
This commit is contained in:
Research Assistant
parent
cf20075a88
commit
a10bf1fdbd
9 changed files with 460 additions and 167 deletions
|
|
@ -125,6 +125,11 @@ def create_test_vault() -> Path:
|
|||
'journal: "Journal of Shoulder and Elbow Surgery"\n'
|
||||
'impact_factor: "5.2"\n'
|
||||
'category: "骨科"\n'
|
||||
'zotero_key: "TSTONE001"\n'
|
||||
'domain: "骨科"\n'
|
||||
"analyze: true\n"
|
||||
"do_ocr: true\n"
|
||||
'ocr_status: "done"\n'
|
||||
"tags:\n"
|
||||
" - 文献阅读\n"
|
||||
" - 骨科\n"
|
||||
|
|
|
|||
|
|
@ -321,12 +321,12 @@ class TestWorkspacePaths:
|
|||
assert "deep_reading_path" in entry
|
||||
assert "ai_path" in entry
|
||||
|
||||
# Paths should be non-empty strings
|
||||
# Workspace root/main note/ai dir should always exist; file-backed paths only when assets exist
|
||||
assert isinstance(entry["paper_root"], str) and len(entry["paper_root"]) > 0
|
||||
assert isinstance(entry["main_note_path"], str) and len(entry["main_note_path"]) > 0
|
||||
assert isinstance(entry["fulltext_path"], str) and len(entry["fulltext_path"]) > 0
|
||||
assert isinstance(entry["deep_reading_path"], str) and len(entry["deep_reading_path"]) > 0
|
||||
assert isinstance(entry["ai_path"], str) and len(entry["ai_path"]) > 0
|
||||
assert entry["fulltext_path"] == ""
|
||||
assert entry["deep_reading_path"] == ""
|
||||
|
||||
def test_path_fields_consistent_after_incremental(self, tmp_path: Path) -> None:
|
||||
"""After incremental refresh, the targeted entry has consistent path fields."""
|
||||
|
|
@ -359,6 +359,43 @@ class TestWorkspacePaths:
|
|||
assert "\\" not in entry["deep_reading_path"]
|
||||
assert "\\" not in entry["ai_path"]
|
||||
|
||||
def test_file_backed_paths_only_present_when_workspace_assets_exist(self, tmp_path: Path) -> None:
|
||||
"""fulltext/deep-reading paths should only be advertised when the files exist."""
|
||||
from paperforge.worker.asset_index import read_index, refresh_index_entry
|
||||
|
||||
items = [_make_export_item("AAA", "Paper A")]
|
||||
items[0]["attachments"] = [{"contentType": "application/pdf", "path": "storage:AAA/paper.pdf"}]
|
||||
vault = _setup_incremental_vault(tmp_path, items)
|
||||
paths = _pipeline_paths(vault)
|
||||
|
||||
zotero_pdf = vault / "99_System" / "Zotero" / "storage" / "AAA"
|
||||
zotero_pdf.mkdir(parents=True, exist_ok=True)
|
||||
(zotero_pdf / "paper.pdf").write_text("pdf", encoding="utf-8")
|
||||
|
||||
ocr_dir = paths["ocr"] / "AAA"
|
||||
ocr_dir.mkdir(parents=True, exist_ok=True)
|
||||
(ocr_dir / "meta.json").write_text(
|
||||
json.dumps({"zotero_key": "AAA", "ocr_status": "done", "page_count": 1}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
workspace_dir = paths["literature"] / "test_domain" / "AAA - Paper A"
|
||||
workspace_dir.mkdir(parents=True, exist_ok=True)
|
||||
(workspace_dir / "AAA - Paper A.md").write_text(
|
||||
"---\ntitle: Paper A\nzotero_key: AAA\n---\n\n## 🔍 精读\n\nDone.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(workspace_dir / "deep-reading.md").write_text("## 🔍 精读\n\nDone.\n", encoding="utf-8")
|
||||
|
||||
_write_index(vault, [{"zotero_key": "AAA", "title": "Paper A"}])
|
||||
refresh_index_entry(vault, "AAA")
|
||||
|
||||
entry = read_index(vault)["items"][0]
|
||||
assert entry["deep_reading_path"].endswith("deep-reading.md")
|
||||
assert entry["fulltext_path"] == ""
|
||||
assert entry["lifecycle"] == "pdf_ready"
|
||||
assert entry["ocr_status"] == "done_incomplete"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests: Derived state fields (Phase 24)
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ class TestSelectionSyncCompletesCleanly:
|
|||
assert "deep_reading_status:" in text
|
||||
assert "first_author:" in text
|
||||
assert "journal:" in text
|
||||
assert "fulltext_md_path:" in text
|
||||
|
||||
def test_formal_note_pdf_wikilink_is_valid(self, test_vault: Path) -> None:
|
||||
"""Verify pdf_path is a valid Obsidian wikilink with forward slashes."""
|
||||
|
|
|
|||
|
|
@ -272,12 +272,136 @@ class TestMigrateToWorkspace:
|
|||
assert marker.read_text(encoding="utf-8") == "do-not-overwrite"
|
||||
|
||||
def test_migrate_returns_zero_when_no_index(self, tmp_path: Path) -> None:
|
||||
"""No index file means nothing to migrate — returns 0."""
|
||||
"""Legacy flat notes still migrate even before the canonical index exists."""
|
||||
vault = _minimal_vault(tmp_path)
|
||||
_ensure_domain_config(vault)
|
||||
# Do NOT write an index
|
||||
|
||||
content = (
|
||||
"---\n"
|
||||
"title: Legacy Flat Note\n"
|
||||
"zotero_key: LEG001\n"
|
||||
"do_ocr: false\n"
|
||||
"analyze: false\n"
|
||||
"---\n\n"
|
||||
"# Legacy Flat Note\n\n"
|
||||
"## 🔍 精读\n\n"
|
||||
"Legacy deep-reading content.\n"
|
||||
)
|
||||
_create_flat_note(vault, "LEG001", "骨科", "Legacy Flat Note", content)
|
||||
|
||||
count = self._call_migrate(vault)
|
||||
assert count == 1
|
||||
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
|
||||
paths = pipeline_paths(vault)
|
||||
workspace_dir = paths["literature"] / "骨科" / "LEG001 - Legacy Flat Note"
|
||||
assert workspace_dir.is_dir()
|
||||
assert (workspace_dir / "LEG001 - Legacy Flat Note.md").exists()
|
||||
assert (workspace_dir / "deep-reading.md").exists()
|
||||
|
||||
def test_migrate_promotes_legacy_library_record_flags(self, tmp_path: Path) -> None:
|
||||
"""Legacy library-record control flags are copied into the workspace note."""
|
||||
vault = _minimal_vault(tmp_path)
|
||||
_ensure_domain_config(vault)
|
||||
|
||||
content = "---\ntitle: Legacy Flags\nzotero_key: LEG002\ndo_ocr: false\nanalyze: false\n---\n\n# Legacy Flags\n"
|
||||
_create_flat_note(vault, "LEG002", "骨科", "Legacy Flags", content)
|
||||
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
|
||||
paths = pipeline_paths(vault)
|
||||
records_dir = paths["library_records"] / "骨科"
|
||||
records_dir.mkdir(parents=True, exist_ok=True)
|
||||
(records_dir / "LEG002.md").write_text(
|
||||
"---\nzotero_key: LEG002\ndo_ocr: true\nanalyze: true\n---\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
count = self._call_migrate(vault)
|
||||
assert count == 1
|
||||
|
||||
workspace_note = paths["literature"] / "骨科" / "LEG002 - Legacy Flags" / "LEG002 - Legacy Flags.md"
|
||||
note_text = workspace_note.read_text(encoding="utf-8")
|
||||
assert 'do_ocr: "true"' in note_text
|
||||
assert 'analyze: "true"' in note_text
|
||||
|
||||
def test_migrate_legacy_record_false_overrides_stale_true(self, tmp_path: Path) -> None:
|
||||
"""Legacy library-record false values should override stale true flags in flat notes."""
|
||||
vault = _minimal_vault(tmp_path)
|
||||
_ensure_domain_config(vault)
|
||||
|
||||
content = "---\ntitle: Legacy False Flags\nzotero_key: LEG003\ndo_ocr: true\nanalyze: true\n---\n\n# Legacy False Flags\n"
|
||||
_create_flat_note(vault, "LEG003", "骨科", "Legacy False Flags", content)
|
||||
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
|
||||
paths = pipeline_paths(vault)
|
||||
records_dir = paths["library_records"] / "骨科"
|
||||
records_dir.mkdir(parents=True, exist_ok=True)
|
||||
(records_dir / "LEG003.md").write_text(
|
||||
"---\nzotero_key: LEG003\ndo_ocr: false\nanalyze: false\n---\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
count = self._call_migrate(vault)
|
||||
assert count == 1
|
||||
|
||||
workspace_note = paths["literature"] / "骨科" / "LEG003 - Legacy False Flags" / "LEG003 - Legacy False Flags.md"
|
||||
note_text = workspace_note.read_text(encoding="utf-8")
|
||||
assert 'do_ocr: "false"' in note_text
|
||||
assert 'analyze: "false"' in note_text
|
||||
|
||||
def test_migrate_uses_frontmatter_title_when_filename_is_noncanonical(self, tmp_path: Path) -> None:
|
||||
"""Legacy files named only by key should still migrate via frontmatter title."""
|
||||
vault = _minimal_vault(tmp_path)
|
||||
_ensure_domain_config(vault)
|
||||
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
|
||||
paths = pipeline_paths(vault)
|
||||
lit_dir = paths["literature"] / "骨科"
|
||||
lit_dir.mkdir(parents=True, exist_ok=True)
|
||||
legacy_path = lit_dir / "LEG004.md"
|
||||
legacy_path.write_text(
|
||||
"---\ntitle: Noncanonical Legacy Name\nzotero_key: LEG004\n---\n\n# Noncanonical Legacy Name\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
count = self._call_migrate(vault)
|
||||
assert count == 1
|
||||
|
||||
workspace_note = paths["literature"] / "骨科" / "LEG004 - Noncanonical Legacy Name" / "LEG004 - Noncanonical Legacy Name.md"
|
||||
assert workspace_note.exists()
|
||||
|
||||
def test_migrate_reconciles_existing_workspace_flags_from_legacy_records(self, tmp_path: Path) -> None:
|
||||
"""Already-migrated workspace notes are reconciled when they still mirror stale flat-note flags."""
|
||||
vault = _minimal_vault(tmp_path)
|
||||
_ensure_domain_config(vault)
|
||||
|
||||
content = "---\ntitle: Existing Workspace\nzotero_key: LEG005\ndo_ocr: true\nanalyze: true\n---\n\n# Existing Workspace\n"
|
||||
_create_flat_note(vault, "LEG005", "骨科", "Existing Workspace", content)
|
||||
|
||||
from paperforge.worker._utils import pipeline_paths
|
||||
|
||||
paths = pipeline_paths(vault)
|
||||
workspace_dir = paths["literature"] / "骨科" / "LEG005 - Existing Workspace"
|
||||
workspace_dir.mkdir(parents=True, exist_ok=True)
|
||||
workspace_note = workspace_dir / "LEG005 - Existing Workspace.md"
|
||||
workspace_note.write_text(content, encoding="utf-8")
|
||||
|
||||
records_dir = paths["library_records"] / "骨科"
|
||||
records_dir.mkdir(parents=True, exist_ok=True)
|
||||
(records_dir / "LEG005.md").write_text(
|
||||
"---\nzotero_key: LEG005\ndo_ocr: false\nanalyze: false\n---\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
count = self._call_migrate(vault)
|
||||
assert count == 0
|
||||
updated_text = workspace_note.read_text(encoding="utf-8")
|
||||
assert 'do_ocr: "false"' in updated_text
|
||||
assert 'analyze: "false"' in updated_text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -126,3 +126,85 @@ def test_render_page_blocks_links_media_for_text_caption(tmp_path: Path) -> None
|
|||
|
||||
assert any(line.startswith("![[") for line in rendered)
|
||||
assert any(line.startswith("Figure 4 RT-qPCR results.") for line in rendered)
|
||||
|
||||
|
||||
def test_embedded_figure_text_excludes_body_paragraph_like_text() -> None:
|
||||
from paperforge.worker.ocr import is_embedded_figure_text_block
|
||||
|
||||
blocks = [
|
||||
{
|
||||
"block_id": 1,
|
||||
"block_label": "paragraph_title",
|
||||
"block_bbox": [80, 530, 269, 554],
|
||||
"block_content": "Immunohistochemistry",
|
||||
},
|
||||
{
|
||||
"block_id": 2,
|
||||
"block_label": "text",
|
||||
"block_bbox": [78, 577, 498, 915],
|
||||
"block_content": (
|
||||
"To determine which cells express these proteins, we performed immunohistochemical analyses "
|
||||
"for Tyk2, S100A9, and ZNF 217 from the same tissues used in 2-DE. Positively stained cells "
|
||||
"of Tyk2 and ZNF 217 were located at cytoplasm of epithelial cells. Although the majority of "
|
||||
"cases of SCC and adjacent normal tissues showed cytoplasmic S100A9 positivity of epithelial "
|
||||
"cells, some cases also showed nuclear S100A9 positivity, as shown in Fig. 3."
|
||||
),
|
||||
},
|
||||
{
|
||||
"block_id": 3,
|
||||
"block_label": "image",
|
||||
"block_bbox": [272, 964, 610, 1421],
|
||||
"block_content": "",
|
||||
},
|
||||
{
|
||||
"block_id": 4,
|
||||
"block_label": "image",
|
||||
"block_bbox": [620, 964, 933, 1421],
|
||||
"block_content": "",
|
||||
},
|
||||
{
|
||||
"block_id": 5,
|
||||
"block_label": "figure_title",
|
||||
"block_bbox": [78, 1448, 1129, 1511],
|
||||
"block_content": (
|
||||
"Fig. 3. Immunohistochemical staining in SCC and adjacent normal tissues. Sections from SCC and "
|
||||
"nontumorous tissue were immunostained with antibodies against Tyk2 (left), S100A9 (middle), "
|
||||
"and ZNF 217 (right)."
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
assert is_embedded_figure_text_block(blocks[1], blocks, page_width=1200, page_height=1600) is False
|
||||
|
||||
|
||||
def test_embedded_figure_text_keeps_narrow_in_figure_note() -> None:
|
||||
from paperforge.worker.ocr import is_embedded_figure_text_block
|
||||
|
||||
blocks = [
|
||||
{
|
||||
"block_id": 1,
|
||||
"block_label": "chart",
|
||||
"block_bbox": [320, 260, 620, 1080],
|
||||
"block_content": "",
|
||||
},
|
||||
{
|
||||
"block_id": 2,
|
||||
"block_label": "chart",
|
||||
"block_bbox": [640, 260, 920, 1080],
|
||||
"block_content": "",
|
||||
},
|
||||
{
|
||||
"block_id": 3,
|
||||
"block_label": "text",
|
||||
"block_bbox": [470, 1090, 760, 1160],
|
||||
"block_content": "Patient A\nAge 42\nHPV16 positive",
|
||||
},
|
||||
{
|
||||
"block_id": 4,
|
||||
"block_label": "figure_title",
|
||||
"block_bbox": [300, 1180, 980, 1250],
|
||||
"block_content": "Figure 2. Representative pathology records and imaging findings.",
|
||||
},
|
||||
]
|
||||
|
||||
assert is_embedded_figure_text_block(blocks[2], blocks, page_width=1200, page_height=1600) is True
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ sync_ocr_queue state reconciliation, and cleanup_blocked_ocr_dirs behavior.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -14,6 +16,26 @@ import pytest
|
|||
import requests
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_ocr_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Keep OCR state-machine tests deterministic and offline.
|
||||
|
||||
- No ambient OCR tokens from env/registry
|
||||
- No 15s poll sleeps between cycles
|
||||
- Very small poll budget by default
|
||||
"""
|
||||
|
||||
monkeypatch.delenv("PADDLEOCR_API_TOKEN", raising=False)
|
||||
monkeypatch.delenv("PADDLEOCR_API_TOKEN_USER", raising=False)
|
||||
monkeypatch.setenv("PAPERFORGE_POLL_INTERVAL", "0")
|
||||
monkeypatch.setenv("PAPERFORGE_POLL_MAX_CYCLES", "2")
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"winreg",
|
||||
types.SimpleNamespace(HKEY_CURRENT_USER=object(), OpenKey=lambda *args, **kwargs: (_ for _ in ()).throw(OSError("no registry token"))),
|
||||
)
|
||||
|
||||
|
||||
def _make_vault(tmp_path: Path) -> tuple[Path, Path, Path, Path]:
|
||||
"""Create a minimal vault with all required directories."""
|
||||
vault = tmp_path / "vault"
|
||||
|
|
@ -104,39 +126,40 @@ do_ocr: true
|
|||
mock_post.return_value.json.return_value = {"data": {"jobId": "job-123"}}
|
||||
mock_post.return_value.raise_for_status = MagicMock()
|
||||
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test.pdf", "queue_status": "pending"}
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.ensure_ocr_meta",
|
||||
return_value={"zotero_key": key, "ocr_status": "pending"},
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test.pdf", "queue_status": "pending"}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.ocr.requests.post", mock_post):
|
||||
with patch("paperforge.worker.ocr.requests.get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {
|
||||
"data": {"state": "done", "resultUrl": {"jsonUrl": ""}}
|
||||
}
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch(
|
||||
"paperforge.worker.ocr.ensure_ocr_meta",
|
||||
return_value={"zotero_key": key, "ocr_status": "pending"},
|
||||
):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.ocr.requests.post", mock_post):
|
||||
with patch("paperforge.worker.ocr.requests.get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {
|
||||
"data": {"state": "done", "resultUrl": {"jsonUrl": ""}}
|
||||
}
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
# Check requests.post was called (job submitted)
|
||||
assert mock_post.called, "requests.post was not called - job not submitted"
|
||||
|
|
@ -216,31 +239,32 @@ do_ocr: true
|
|||
|
||||
# Use return_value for all calls — same mock is returned each time
|
||||
# This is sufficient since we only care about the first poll result
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test2.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test2.pdf", "queue_status": "queued"}
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test2.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.ocr.requests.get", return_value=poll_response):
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test2.pdf", "queue_status": "queued"}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.ocr.requests.get", return_value=poll_response):
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [
|
||||
c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key
|
||||
|
|
@ -309,31 +333,32 @@ do_ocr: true
|
|||
error_response.json.return_value = {"data": {"state": "error", "errorMsg": "Model inference failed"}}
|
||||
error_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test3.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test3.pdf", "queue_status": "queued"}
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test3.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.ocr.requests.get", return_value=error_response):
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test3.pdf", "queue_status": "queued"}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.ocr.requests.get", return_value=error_response):
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [
|
||||
c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key
|
||||
|
|
@ -414,38 +439,39 @@ do_ocr: true
|
|||
def make_meta(*args, **kwargs):
|
||||
return {"zotero_key": key, "ocr_status": "pending"}
|
||||
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test4.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with patch("paperforge.worker.ocr.pipeline_paths", return_value=paths):
|
||||
with patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}):
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test4.pdf", "queue_status": "pending"}
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test4.pdf"}],
|
||||
"title": "Test",
|
||||
}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.ensure_ocr_meta", side_effect=make_meta):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
with patch("paperforge.worker.ocr.requests.post", mock_post):
|
||||
with patch("paperforge.worker.ocr.requests.get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {
|
||||
"data": {"state": "done", "resultUrl": {"jsonUrl": ""}}
|
||||
}
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch(
|
||||
"paperforge.worker.ocr.sync_ocr_queue",
|
||||
return_value=[
|
||||
{"zotero_key": key, "has_pdf": True, "pdf_path": "test4.pdf", "queue_status": "pending"}
|
||||
],
|
||||
):
|
||||
with patch("paperforge.worker.ocr.ensure_ocr_meta", side_effect=make_meta):
|
||||
with patch("paperforge.worker.ocr.write_json") as mock_write:
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
with patch("paperforge.worker.ocr.requests.post", mock_post):
|
||||
with patch("paperforge.worker.ocr.requests.get") as mock_get:
|
||||
mock_get.return_value.json.return_value = {
|
||||
"data": {"state": "done", "resultUrl": {"jsonUrl": ""}}
|
||||
}
|
||||
with patch("paperforge.worker.sync.run_selection_sync"):
|
||||
with patch("paperforge.worker.sync.run_index_refresh"):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [
|
||||
c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key
|
||||
|
|
@ -750,26 +776,27 @@ do_ocr: true
|
|||
)
|
||||
(vault / "test.pdf").write_text("PDF content")
|
||||
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch(
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test.pdf"}],
|
||||
"title": "Retry Limit Paper",
|
||||
}
|
||||
],
|
||||
),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch(
|
||||
"paperforge.worker.ocr.load_export_rows",
|
||||
return_value=[
|
||||
{
|
||||
"key": key,
|
||||
"attachments": [{"contentType": "application/pdf", "path": "test.pdf"}],
|
||||
"title": "Retry Limit Paper",
|
||||
}
|
||||
],
|
||||
),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [
|
||||
c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key
|
||||
|
|
@ -870,7 +897,14 @@ class TestOcrEdgeCases:
|
|||
paths = _mock_paths(vault, ocr_root, exports, library_records)
|
||||
import os as _os
|
||||
|
||||
with patch.dict("os.environ", {"PADDLEOCR_MAX_ITEMS": "1", **TestOcrEdgeCases.SHORT_RETRY}):
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"PADDLEOCR_MAX_ITEMS": "1",
|
||||
"PADDLEOCR_API_TOKEN": "test-token",
|
||||
**TestOcrEdgeCases.SHORT_RETRY,
|
||||
},
|
||||
):
|
||||
for idx in range(4):
|
||||
key = f"KEY_M1_{idx}"
|
||||
meta_path = ocr_root / key / "meta.json"
|
||||
|
|
@ -895,6 +929,8 @@ class TestOcrEdgeCases:
|
|||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={f"KEY_M1_{i}": {"do_ocr": True} for i in range(4)}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": f"KEY_M1_{i}", "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"} for i in range(4)]),
|
||||
patch("paperforge.worker.ocr.requests.post", mock_post),
|
||||
patch("paperforge.worker.ocr.requests.get", mock_get),
|
||||
patch("paperforge.worker.ocr.write_json"),
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
|
|
@ -1015,18 +1051,19 @@ class TestOcrEdgeCases:
|
|||
|
||||
mock_post = MagicMock(side_effect=requests.exceptions.Timeout("request timed out"))
|
||||
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]),
|
||||
patch("paperforge.worker.ocr.requests.post", mock_post),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token", **TestOcrEdgeCases.SHORT_RETRY}):
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]),
|
||||
patch("paperforge.worker.ocr.requests.post", mock_post),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key]
|
||||
assert meta_calls, "No meta write after upload timeout"
|
||||
|
|
@ -1059,19 +1096,20 @@ class TestOcrEdgeCases:
|
|||
mock_get.return_value.json.return_value = {"data": {"state": "bizarre_unknown", "resultUrl": {"jsonUrl": "http://x.com"}}}
|
||||
mock_get.return_value.raise_for_status = MagicMock()
|
||||
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]),
|
||||
patch("paperforge.worker.ocr.requests.post", mock_post),
|
||||
patch("paperforge.worker.ocr.requests.get", mock_get),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]),
|
||||
patch("paperforge.worker.ocr.requests.post", mock_post),
|
||||
patch("paperforge.worker.ocr.requests.get", mock_get),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key]
|
||||
assert meta_calls, "No meta write for unknown state item"
|
||||
|
|
@ -1123,28 +1161,34 @@ class TestOcrEdgeCases:
|
|||
mock_post.return_value.raise_for_status = MagicMock()
|
||||
|
||||
mock_get_poll = MagicMock()
|
||||
mock_get_poll.return_value.json.return_value = {"data": {"state": "done", "resultUrl": {"jsonUrl": "http://fake.url/result"}}}
|
||||
mock_get_poll.return_value.raise_for_status = MagicMock()
|
||||
mock_get_poll.json.return_value = {"data": {"state": "done", "resultUrl": {"jsonUrl": "http://fake.url/result"}}}
|
||||
mock_get_poll.raise_for_status = MagicMock()
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.text = result_text
|
||||
mock_result.status_code = 200
|
||||
mock_result.raise_for_status = MagicMock()
|
||||
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
with patch("paperforge.worker.ocr.requests.post", mock_post):
|
||||
with patch("paperforge.worker.ocr.requests.get") as mock_get_all:
|
||||
mock_get_all.side_effect = [mock_get_poll, mock_result]
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
def _write_through_json(path: Path, data: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
run_ocr(vault)
|
||||
with patch.dict("os.environ", {"PADDLEOCR_API_TOKEN": "test-token"}):
|
||||
with (
|
||||
patch("paperforge.worker.ocr.pipeline_paths", return_value=paths),
|
||||
patch("paperforge.worker.ocr.load_control_actions", return_value={key: {"do_ocr": True}}),
|
||||
patch("paperforge.worker.ocr.load_export_rows", return_value=[{"key": key, "attachments": [{"contentType": "application/pdf", "path": "test.pdf"}], "title": "T"}]),
|
||||
patch("paperforge.worker.ocr.write_json") as mock_write,
|
||||
patch("paperforge.worker.sync.run_selection_sync"),
|
||||
patch("paperforge.worker.sync.run_index_refresh"),
|
||||
):
|
||||
with patch("paperforge.worker.ocr.requests.post", mock_post):
|
||||
with patch("paperforge.worker.ocr.requests.get") as mock_get_all:
|
||||
mock_get_all.side_effect = [mock_get_poll, mock_result]
|
||||
mock_write.side_effect = _write_through_json
|
||||
from paperforge.worker.ocr import run_ocr
|
||||
|
||||
run_ocr(vault)
|
||||
|
||||
meta_calls = [c for c in mock_write.call_args_list if isinstance(c[0][1], dict) and c[0][1].get("zotero_key") == key]
|
||||
assert meta_calls
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ class TestCanonicalIndexOcrState:
|
|||
entry = next((e for e in index.get("items", []) if e.get("zotero_key") == "VALID"), None)
|
||||
assert entry is not None, "VALID should be in canonical index"
|
||||
assert entry["has_pdf"] is True
|
||||
assert entry["do_ocr"] is True
|
||||
assert entry["do_ocr"] is False # do_ocr defaults to False (user-controlled)
|
||||
assert entry["ocr_status"] == "pending"
|
||||
|
||||
def test_pdf_with_ocr_done_meta(self, tmp_path: Path) -> None:
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ class TestEnvCheckerInit:
|
|||
def test_init_with_vault_path(self, tmp_path: Path) -> None:
|
||||
checker = EnvChecker(tmp_path)
|
||||
assert checker.vault == tmp_path
|
||||
assert checker.system_dir == "99_System"
|
||||
assert checker.system_dir == "System"
|
||||
assert set(checker.results.keys()) == {"python", "vault", "zotero", "bbt", "json"}
|
||||
|
||||
def test_results_are_checkresult_instances(self, tmp_path: Path) -> None:
|
||||
|
|
@ -152,7 +152,7 @@ class TestEnvCheckerInit:
|
|||
|
||||
def test_get_exports_dir(self, tmp_path: Path) -> None:
|
||||
checker = EnvChecker(tmp_path)
|
||||
expected = tmp_path / "99_System" / "PaperForge" / "exports"
|
||||
expected = tmp_path / "System" / "PaperForge" / "exports"
|
||||
assert checker.get_exports_dir() == expected
|
||||
|
||||
def test_get_exports_dir_custom_system_dir(self, tmp_path: Path) -> None:
|
||||
|
|
@ -192,15 +192,15 @@ class TestEnvCheckerCheckVault:
|
|||
|
||||
def test_existing_directories_passes(self, tmp_path: Path) -> None:
|
||||
"""Vault with correct structure should pass."""
|
||||
(tmp_path / "99_System" / "PaperForge" / "exports").mkdir(parents=True)
|
||||
(tmp_path / "99_System" / "PaperForge" / "ocr").mkdir(parents=True)
|
||||
(tmp_path / "System" / "PaperForge" / "exports").mkdir(parents=True)
|
||||
(tmp_path / "System" / "PaperForge" / "ocr").mkdir(parents=True)
|
||||
checker = EnvChecker(tmp_path)
|
||||
result = checker.check_vault()
|
||||
assert result.passed is True
|
||||
|
||||
def test_partial_directories_fails(self, tmp_path: Path) -> None:
|
||||
"""Vault with only one required dir should fail."""
|
||||
(tmp_path / "99_System" / "PaperForge" / "exports").mkdir(parents=True)
|
||||
(tmp_path / "System" / "PaperForge" / "exports").mkdir(parents=True)
|
||||
checker = EnvChecker(tmp_path)
|
||||
result = checker.check_vault()
|
||||
assert result.passed is False
|
||||
|
|
@ -292,7 +292,7 @@ class TestEnvCheckerCheckJson:
|
|||
assert result.passed is False
|
||||
|
||||
def test_valid_json_exports_pass(self, tmp_path: Path) -> None:
|
||||
exports_dir = tmp_path / "99_System" / "PaperForge" / "exports"
|
||||
exports_dir = tmp_path / "System" / "PaperForge" / "exports"
|
||||
exports_dir.mkdir(parents=True)
|
||||
(exports_dir / "test.json").write_text(
|
||||
json.dumps({"items": [{"key": "TEST"}]}),
|
||||
|
|
@ -304,7 +304,7 @@ class TestEnvCheckerCheckJson:
|
|||
assert "JSON" in result.detail
|
||||
|
||||
def test_invalid_json_fails(self, tmp_path: Path) -> None:
|
||||
exports_dir = tmp_path / "99_System" / "PaperForge" / "exports"
|
||||
exports_dir = tmp_path / "System" / "PaperForge" / "exports"
|
||||
exports_dir.mkdir(parents=True)
|
||||
(exports_dir / "bad.json").write_text("{invalid}", encoding="utf-8")
|
||||
checker = EnvChecker(tmp_path)
|
||||
|
|
@ -312,7 +312,7 @@ class TestEnvCheckerCheckJson:
|
|||
assert result.passed is False
|
||||
|
||||
def test_empty_exports_fails(self, tmp_path: Path) -> None:
|
||||
exports_dir = tmp_path / "99_System" / "PaperForge" / "exports"
|
||||
exports_dir = tmp_path / "System" / "PaperForge" / "exports"
|
||||
exports_dir.mkdir(parents=True)
|
||||
checker = EnvChecker(tmp_path)
|
||||
result = checker.check_json()
|
||||
|
|
|
|||
|
|
@ -63,9 +63,9 @@ class TestSetupWizard:
|
|||
|
||||
def test_setup_wizard_pip_install(self, test_vault: Path) -> None:
|
||||
"""Verify setup wizard's _deploy method calls pip install -e ."""
|
||||
import setup_wizard
|
||||
from paperforge.setup_wizard import headless_setup
|
||||
|
||||
source = Path(setup_wizard.__file__).read_text(encoding="utf-8")
|
||||
source = Path(headless_setup.__code__.co_filename).read_text(encoding="utf-8")
|
||||
assert (
|
||||
'"-m", "pip", "install", "-e"' in source or "pip install -e" in source
|
||||
), "setup wizard should call pip install -e"
|
||||
|
|
|
|||
Loading…
Reference in a new issue