diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index 169d753e..a17fe52a 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -1805,7 +1805,8 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu metadata_dir = ocr_root / "metadata" metadata_dir.mkdir(parents=True, exist_ok=True) frontmatter_candidates = extract_frontmatter_candidates(artifacts.blocks_structured) - resolved = resolve_metadata(source_meta, frontmatter_candidates) + page1_raw = [b for b in all_raw_blocks if b.get("page") == 1] if all_raw_blocks else None + resolved = resolve_metadata(source_meta, frontmatter_candidates, page1_blocks=page1_raw) write_resolved_metadata(metadata_dir / "resolved_metadata.json", resolved) # --- Phase 2: figure inventory --- @@ -1871,6 +1872,7 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu structured_blocks=structured, figure_inventory=figure_inventory, table_inventory=table_inventory, + doc_structure=doc_structure, ) write_ocr_health(ocr_root / "health", health_report) meta["ocr_health_overall"] = health_report["overall"] diff --git a/paperforge/worker/ocr_artifacts.py b/paperforge/worker/ocr_artifacts.py index f3e15b1f..62f06523 100644 --- a/paperforge/worker/ocr_artifacts.py +++ b/paperforge/worker/ocr_artifacts.py @@ -72,3 +72,38 @@ def compute_json_hash(data: list | dict) -> str: if isinstance(data, list): data = {"data": data} return _sha256_hexdigest(json.dumps(data, sort_keys=True).encode("utf-8")) + + +def cleanup_ocr_artifact_cache(paper_root: Path, *, dry_run: bool = False) -> dict[str, Any]: + """Remove regenerable cache artifacts while preserving canonical data. + + Canonical (kept): + canonical/, structure/, metadata/, assets/, render/, health/, index/ + raw/, meta.json, fulltext.md + + Cache (removed): + pages/ — page render cache, always regenerable from source PDF + + Returns a summary dict with paths removed per category. + """ + report: dict[str, Any] = {"pages_removed": [], "errors": []} + + pages_dir = paper_root / "pages" + if pages_dir.is_dir(): + for f in sorted(pages_dir.iterdir()): + if f.suffix in {".jpg", ".png", ".webp"}: + if not dry_run: + try: + f.unlink() + except OSError as e: + report["errors"].append(str(e)) + report["pages_removed"].append(f.name) + if not dry_run: + try: + remaining = list(pages_dir.iterdir()) + if not remaining: + pages_dir.rmdir() + except OSError: + pass + + return report diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 1247d48f..951bd9e4 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -403,6 +403,25 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 if i not in used_asset_indices: unmatched_assets.append(asset) + # Build unresolved clusters: spatial clusters of unmatched assets on + # pages where all candidate legends were rejected (multi-panel figures + # with axis labels or informal captions) + if rejected_legends and unmatched_assets: + rejected_pages = {leg.get("page") for leg in rejected_legends if leg.get("page")} + for cluster in _media_clusters(unmatched_assets, page_width): + cluster_page = cluster[0].get("page", 0) + if cluster_page not in rejected_pages: + continue + cluster_ids = [b.get("block_id", "") for b in cluster] + unresolved_clusters.append({ + "media_block_ids": cluster_ids, + "cluster_bbox": _cluster_bbox([b.get("bbox", [0, 0, 0, 0]) for b in cluster]), + "page": cluster_page, + }) + if unresolved_clusters: + consumed = {bid for uc in unresolved_clusters for bid in uc["media_block_ids"]} + unmatched_assets = [a for a in unmatched_assets if a.get("block_id", "") not in consumed] + return { "figure_legends": legends, "figure_assets": assets, diff --git a/paperforge/worker/ocr_health.py b/paperforge/worker/ocr_health.py index 8f2421dc..d502b368 100644 --- a/paperforge/worker/ocr_health.py +++ b/paperforge/worker/ocr_health.py @@ -11,6 +11,7 @@ def build_ocr_health( structured_blocks: list[dict], figure_inventory: dict, table_inventory: dict, + doc_structure: Any = None, ) -> dict[str, Any]: section_heading_count = sum(1 for b in structured_blocks if b.get("role") == "section_heading") abstract_found = any( @@ -64,7 +65,7 @@ def build_ocr_health( from paperforge.worker.ocr_document import _compute_span_coverage, _detect_body_spine, _run_layout_audit span = _compute_span_coverage(structured_blocks) - spine = _detect_body_spine(structured_blocks) + spine = _detect_body_spine(structured_blocks, doc=doc_structure) layout = _run_layout_audit(structured_blocks) return { diff --git a/paperforge/worker/ocr_rebuild.py b/paperforge/worker/ocr_rebuild.py index a91aa361..e944cdf7 100644 --- a/paperforge/worker/ocr_rebuild.py +++ b/paperforge/worker/ocr_rebuild.py @@ -185,6 +185,7 @@ def run_derived_rebuild_for_keys(vault: Path, keys: list[str]) -> dict: structured_blocks=structured, figure_inventory=figure_inventory, table_inventory=table_inventory, + doc_structure=doc_structure, ) write_ocr_health(paper_root / "health", health_report) diff --git a/tests/test_ocr_artifacts.py b/tests/test_ocr_artifacts.py index 48ca271c..69f5df99 100644 --- a/tests/test_ocr_artifacts.py +++ b/tests/test_ocr_artifacts.py @@ -28,3 +28,40 @@ def test_raw_and_derived_version_payloads_have_separate_namespaces() -> None: assert "derived_version" in payload assert payload["raw_version"]["ocr_model"] == "PaddleOCR-VL-1.6" assert "renderer_version" in payload["derived_version"] + + +def test_cleanup_ocr_cache_removes_page_cache_files(tmp_path: Path) -> None: + from paperforge.worker.ocr_artifacts import cleanup_ocr_artifact_cache + + paper_root = tmp_path / "paper" + pages_dir = paper_root / "pages" + pages_dir.mkdir(parents=True) + (pages_dir / "page_001.jpg").write_text("fake jpg", encoding="utf-8") + (pages_dir / "page_002.png").write_text("fake png", encoding="utf-8") + # Canonical data must NOT be touched + canonical_dir = paper_root / "canonical" + canonical_dir.mkdir(parents=True) + (canonical_dir / "blocks.raw.jsonl").write_text("{}", encoding="utf-8") + + report = cleanup_ocr_artifact_cache(paper_root) + + assert len(report["pages_removed"]) == 2 + assert "page_001.jpg" in report["pages_removed"] + assert "page_002.png" in report["pages_removed"] + assert not pages_dir.exists(), "pages dir should be removed when empty" + assert canonical_dir.exists(), "canonical data must survive" + + +def test_cleanup_ocr_cache_dry_run_does_not_delete(tmp_path: Path) -> None: + from paperforge.worker.ocr_artifacts import cleanup_ocr_artifact_cache + + paper_root = tmp_path / "paper" + pages_dir = paper_root / "pages" + pages_dir.mkdir(parents=True) + (pages_dir / "page_001.jpg").write_text("fake", encoding="utf-8") + + report = cleanup_ocr_artifact_cache(paper_root, dry_run=True) + + assert len(report["pages_removed"]) == 1 + assert pages_dir.exists(), "dry run must not delete files" + assert (pages_dir / "page_001.jpg").exists(), "dry run must not delete files"