"""Tests for OCR state in canonical index entries. Verifies that build_index() produces correct OCR status and workflow fields in canonical index entries for papers with/without PDFs. """ from __future__ import annotations import json from pathlib import Path from unittest.mock import patch # Pre-import ocr module so lazy imports inside build_index() bind to the # real pipeline_paths, not to any mock that may be active during the test. from paperforge.worker import ocr as _preimport_ocr # noqa: F401 class TestCanonicalIndexOcrState: """Tests for OCR state reporting in canonical index.""" def _make_vault(self, tmp_path: Path) -> Path: """Create a minimal vault structure.""" vault = tmp_path / "vault" vault.mkdir() system = vault / "99_System" system.mkdir() exports = system / "PaperForge" / "exports" exports.mkdir(parents=True) (exports / "test.json").write_text("[]", encoding="utf-8") (system / "PaperForge" / "ocr").mkdir(parents=True) (system / "PaperForge" / "indexes").mkdir(parents=True) resources = vault / "03_Resources" resources.mkdir() (resources / "Literature").mkdir(parents=True) (vault / "paperforge.json").write_text( json.dumps( { "vault_config": { "system_dir": "99_System", "resources_dir": "03_Resources", "literature_dir": "Literature", "control_dir": "LiteratureControl", "base_dir": "05_Bases", } } ), encoding="utf-8", ) return vault def _mock_export_item(self, key: str, has_pdf: bool = True, pdf_path: str = "") -> dict: """Build a mock export item dict.""" attachments = [] if has_pdf and pdf_path: attachments.append({"contentType": "application/pdf", "path": pdf_path}) return { "key": key, "title": "Test Paper", "year": 2024, "doi": "10.1234/test", "attachments": attachments, "collections": [], "creators": [], "publicationTitle": "", "journal": "Test Journal", "extra": "", "authors": ["Smith J"], } def _base_mock_paths(self, vault: Path) -> dict: return { "vault": vault, "exports": vault / "99_System" / "PaperForge" / "exports", "ocr": vault / "99_System" / "PaperForge" / "ocr", "literature": vault / "03_Resources" / "Literature", "index": vault / "99_System" / "PaperForge" / "indexes" / "formal-library.json", "bases": vault / "05_Bases", "library_records": vault / "03_Resources" / "LiteratureControl" / "library-records", } def test_no_pdf_sets_pending_ocr(self, tmp_path: Path) -> None: """Export item with no PDF → ocr_status stays pending.""" vault = self._make_vault(tmp_path) with ( patch("paperforge.worker._utils.pipeline_paths") as mock_paths, patch("paperforge.worker.sync.load_domain_config") as mock_domain, patch("paperforge.worker.sync.load_export_rows") as mock_export, patch("paperforge.worker.asset_index.migrate_legacy_index", return_value=0), ): mock_export.return_value = [self._mock_export_item("NO_PDF", has_pdf=False)] mock_paths.return_value = self._base_mock_paths(vault) mock_domain.return_value = {"domains": [{"export_file": "test.json", "domain": "TestDomain"}]} from paperforge.worker.asset_index import build_index build_index(vault) index_path = vault / "99_System" / "PaperForge" / "indexes" / "formal-library.json" index = json.loads(index_path.read_text(encoding="utf-8")) entry = next((e for e in index.get("items", []) if e.get("zotero_key") == "NO_PDF"), None) assert entry is not None, "NO_PDF should be in canonical index" assert entry["has_pdf"] is False assert entry["do_ocr"] is False # no PDF → no OCR needed assert entry["ocr_status"] == "pending" def test_pdf_without_ocr_meta_sets_pending(self, tmp_path: Path) -> None: """Export item with PDF but no OCR meta → ocr_status: pending.""" vault = self._make_vault(tmp_path) pdf = tmp_path / "test.pdf" pdf.write_bytes(b"%PDF-1.4\n") with ( patch("paperforge.worker._utils.pipeline_paths") as mock_paths, patch("paperforge.worker.sync.load_domain_config") as mock_domain, patch("paperforge.worker.sync.load_export_rows") as mock_export, patch("paperforge.worker.asset_index.migrate_legacy_index", return_value=0), ): mock_export.return_value = [self._mock_export_item("VALID", has_pdf=True, pdf_path=str(pdf))] mock_paths.return_value = self._base_mock_paths(vault) mock_domain.return_value = {"domains": [{"export_file": "test.json", "domain": "TestDomain"}]} from paperforge.worker.asset_index import build_index build_index(vault) index_path = vault / "99_System" / "PaperForge" / "indexes" / "formal-library.json" index = json.loads(index_path.read_text(encoding="utf-8")) entry = next((e for e in index.get("items", []) if e.get("zotero_key") == "VALID"), None) assert entry is not None, "VALID should be in canonical index" assert entry["has_pdf"] is True assert entry["do_ocr"] is False # do_ocr defaults to False (user-controlled) assert entry["ocr_status"] == "pending" def test_pdf_with_ocr_done_meta(self, tmp_path: Path) -> None: """Export item with PDF and OCR meta.json (ocr_status=done) → index shows done.""" vault = self._make_vault(tmp_path) pdf = tmp_path / "test.pdf" pdf.write_bytes(b"%PDF-1.4\n") ocr_dir = vault / "99_System" / "PaperForge" / "ocr" / "OCR_DONE" ocr_dir.mkdir(parents=True) # Create valid OCR fixture: meta.json, fulltext.md, json/result.json (ocr_dir / "fulltext.md").write_text( "\n" + "Abstract content.\n" * 30 + "\n" + "Methods and results.\n" * 30, encoding="utf-8", ) (ocr_dir / "json").mkdir() (ocr_dir / "json" / "result.json").write_text( json.dumps({"pages": 2, "text": "x" * 1200}), encoding="utf-8", ) (ocr_dir / "meta.json").write_text( json.dumps( { "ocr_status": "done", "zotero_key": "OCR_DONE", "page_count": 2, "markdown_path": str(ocr_dir / "fulltext.md"), } ), encoding="utf-8", ) with ( patch("paperforge.worker._utils.pipeline_paths") as mock_paths, patch("paperforge.worker.sync.load_domain_config") as mock_domain, patch("paperforge.worker.sync.load_export_rows") as mock_export, patch("paperforge.worker.asset_index.migrate_legacy_index", return_value=0), ): mock_export.return_value = [self._mock_export_item("OCR_DONE", has_pdf=True, pdf_path=str(pdf))] mock_paths.return_value = self._base_mock_paths(vault) mock_domain.return_value = {"domains": [{"export_file": "test.json", "domain": "TestDomain"}]} from paperforge.worker.asset_index import build_index build_index(vault) index_path = vault / "99_System" / "PaperForge" / "indexes" / "formal-library.json" index = json.loads(index_path.read_text(encoding="utf-8")) entry = next((e for e in index.get("items", []) if e.get("zotero_key") == "OCR_DONE"), None) assert entry is not None, "OCR_DONE should be in canonical index" assert entry["has_pdf"] is True assert entry["do_ocr"] is True assert entry["ocr_status"] == "done" def test_phase2_artifact_dirs_dont_break_index(tmp_path: Path) -> None: """Verify Phase 2 directories don't break build_index().""" import json from paperforge.worker.asset_index import build_index vault = tmp_path / "vault" vault.mkdir() system = vault / "99_System" system.mkdir() (system / "PaperForge" / "exports").mkdir(parents=True) (system / "PaperForge" / "exports" / "test.json").write_text("[]", encoding="utf-8") (system / "PaperForge" / "ocr").mkdir(parents=True) (system / "PaperForge" / "indexes").mkdir(parents=True) resources = vault / "03_Resources" resources.mkdir() (resources / "Literature").mkdir(parents=True) (vault / "paperforge.json").write_text( json.dumps( { "vault_config": { "system_dir": "99_System", "resources_dir": "03_Resources", "literature_dir": "Literature", "control_dir": "LiteratureControl", "base_dir": "05_Bases", } } ), encoding="utf-8", ) # Simulate a paper with Phase 2 artifact dirs ocr_dir = system / "PaperForge" / "ocr" / "PHASE2CK" ocr_dir.mkdir(parents=True) (ocr_dir / "metadata").mkdir() (ocr_dir / "metadata" / "resolved_metadata.json").write_text("{}", encoding="utf-8") (ocr_dir / "structure").mkdir() (ocr_dir / "structure" / "figure_inventory.json").write_text("{}", encoding="utf-8") (ocr_dir / "structure" / "table_inventory.json").write_text("{}", encoding="utf-8") (ocr_dir / "assets" / "figures").mkdir(parents=True) (ocr_dir / "assets" / "tables").mkdir(parents=True) (ocr_dir / "assets" / "orphans").mkdir(parents=True) (ocr_dir / "render" / "figures").mkdir(parents=True) (ocr_dir / "render" / "tables").mkdir(parents=True) (ocr_dir / "meta.json").write_text('{"zotero_key":"PHASE2CK","ocr_status":"done"}', encoding="utf-8") # build_index should not raise or fail on Phase 2 directories rc = build_index(vault) assert rc == 0, f"build_index failed with rc={rc}" def test_sync_reads_enriched_meta_without_breaking_ocr_status(tmp_path: Path) -> None: """Ensure sync tolerates enriched meta.json with raw_version + derived_version.""" import json from paperforge.config import paperforge_paths from paperforge.worker.ocr import validate_ocr_meta ocr_root = tmp_path / "System" / "PaperForge" / "ocr" / "TESTKEY" ocr_root.mkdir(parents=True) # Write enriched meta.json with Phase 1 version fields meta = { "zotero_key": "TESTKEY", "ocr_status": "done", "ocr_provider": "PaddleOCR-VL-1.6", "page_count": 5, "markdown_path": "System/PaperForge/ocr/TESTKEY/fulltext.md", "json_path": "System/PaperForge/ocr/TESTKEY/json/result.json", "fulltext_md_path": str(ocr_root / "fulltext.md"), "raw_version": { "ocr_provider": "PaddleOCR", "ocr_model": "PaddleOCR-VL-1.6", "ocr_raw_schema_version": "1.0.0", "pdf_fingerprint": "sha256:abc", "result_json_hash": "sha256:def", }, "derived_version": { "canonical_block_version": "1.0.0", "structure_version": "1.0.0", "metadata_resolver_version": "0.0.0-phase1", "asset_extractor_version": "0.0.0-phase1", "renderer_version": "1.0.0-compat", "doctor_version": "0.0.0-phase1", }, } (ocr_root / "meta.json").write_text(json.dumps(meta), encoding="utf-8") # Write required compatibility files with valid sizes (ocr_root / "fulltext.md").write_text( "\n" + "A" * 700 + "\n\n" + "B" * 700 + "\n\n" + "C" * 700 + "\n\n" + "D" * 700 + "\n\n" + "E" * 700, encoding="utf-8", ) (ocr_root / "json").mkdir(exist_ok=True) result_data = {"pages": 5, "blocks": [{"id": i, "text": "x" * 100} for i in range(30)]} import json as _json (ocr_root / "json" / "result.json").write_text(_json.dumps(result_data), encoding="utf-8") paths = paperforge_paths(tmp_path) status, error = validate_ocr_meta(paths, meta) assert status == "done" assert error == "" def test_postprocess_writes_compat_fulltext_at_top_level(tmp_path) -> None: """Phase 3 compatibility: top-level fulltext.md must still exist.""" import json as _json from paperforge.worker.ocr import postprocess_ocr_result vault = tmp_path / "vault" vault.mkdir() (vault / "paperforge.json").write_text( _json.dumps({"vault_config": {"system_dir": "System", "resources_dir": "Resources"}}), encoding="utf-8", ) ocr_root = vault / "System" / "PaperForge" / "ocr" ocr_root.mkdir(parents=True) ocr_dir = ocr_root / "COMP001" ocr_dir.mkdir() (ocr_dir / "meta.json").write_text( '{"zotero_key":"COMP001","ocr_status":"done","ocr_model":"PaddleOCR"}', encoding="utf-8", ) page_num, markdown_path, json_path, fulltext_md_path = postprocess_ocr_result(vault, "COMP001", []) # Top-level fulltext.md must exist assert (ocr_dir / "fulltext.md").exists(), "top-level fulltext.md missing" # Returned path must resolve to the top-level fulltext.md assert "fulltext.md" in fulltext_md_path, "returned path does not point to fulltext.md"