diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index b40e17c2..bca4579c 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -1818,6 +1818,15 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu write_ocr_health(ocr_root / "health", health_report) meta["ocr_health_overall"] = health_report["overall"] + # --- Phase 5: role-based OCR index --- + from paperforge.worker.ocr_index import build_role_indexes, write_role_index + + role_indexes = build_role_indexes( + structured_blocks=structured, + resolved_metadata=resolved, + ) + write_role_index(ocr_root / "index", role_indexes) + # Update meta.json with version payloads ocr_model = meta.get("ocr_model", meta.get("ocr_provider", "PaddleOCR")) version_payload = build_version_payload( diff --git a/paperforge/worker/ocr_index.py b/paperforge/worker/ocr_index.py new file mode 100644 index 00000000..09db3d1d --- /dev/null +++ b/paperforge/worker/ocr_index.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from paperforge.core.io import write_json + + +def build_role_indexes( + structured_blocks: list[dict[str, Any]], + resolved_metadata: dict[str, Any], +) -> dict[str, list[dict[str, Any]]]: + body_roles = { + "body_paragraph", + "section_heading", + "subsection_heading", + "abstract_body", + "abstract_heading", + "introduction_heading", + } + body: list[dict[str, Any]] = [] + captions: list[dict[str, Any]] = [] + tables: list[dict[str, Any]] = [] + references: list[dict[str, Any]] = [] + all_blocks: list[dict[str, Any]] = [] + + for block in structured_blocks: + role = block.get("role", "") + entry = { + "paper_id": block.get("paper_id", ""), + "page": block.get("page", 0), + "block_id": block.get("block_id", ""), + "role": role, + "text": block.get("text", ""), + } + all_blocks.append(entry) + + if role in body_roles: + body.append(entry) + elif role == "figure_caption": + captions.append(entry) + elif role == "table_caption": + tables.append(entry) + elif role == "reference_item": + references.append(entry) + + metadata_index: list[dict[str, Any]] = [] + for key in ("title", "authors", "doi", "journal", "year"): + value = resolved_metadata.get(key, {}).get("value", "") + if value: + metadata_index.append({ + "paper_id": structured_blocks[0].get("paper_id", "") if structured_blocks else "", + "page": 0, + "block_id": f"meta_{key}", + "role": f"metadata_{key}", + "text": str(value), + }) + + return { + "body": body, + "captions": captions, + "tables": tables, + "metadata": metadata_index, + "references": references, + "all_blocks": all_blocks, + } + + +def write_role_index(index_root: Path, indexes: dict[str, Any]) -> None: + index_root.mkdir(parents=True, exist_ok=True) + write_json(index_root / "role-index.json", indexes) diff --git a/paperforge/worker/ocr_rebuild.py b/paperforge/worker/ocr_rebuild.py index 0498e777..94dc85e0 100644 --- a/paperforge/worker/ocr_rebuild.py +++ b/paperforge/worker/ocr_rebuild.py @@ -128,6 +128,15 @@ def run_derived_rebuild_for_keys(vault: Path, keys: list[str]) -> dict: ) write_ocr_health(paper_root / "health", health_report) + # Rebuild role index + from paperforge.worker.ocr_index import build_role_indexes, write_role_index + + role_indexes = build_role_indexes( + structured_blocks=structured, + resolved_metadata=resolved, + ) + write_role_index(paper_root / "index", role_indexes) + # Update version state in meta.json meta = read_json(artifacts.meta_json) if artifacts.meta_json.exists() else {} meta = _apply_post_rebuild_version_flags(meta)