feat: add OCR role-based search index

This commit is contained in:
Research Assistant 2026-06-05 12:13:18 +08:00
parent 4f6b250ca3
commit 5d243c8e72
3 changed files with 89 additions and 0 deletions

View file

@ -1818,6 +1818,15 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu
write_ocr_health(ocr_root / "health", health_report)
meta["ocr_health_overall"] = health_report["overall"]
# --- Phase 5: role-based OCR index ---
from paperforge.worker.ocr_index import build_role_indexes, write_role_index
role_indexes = build_role_indexes(
structured_blocks=structured,
resolved_metadata=resolved,
)
write_role_index(ocr_root / "index", role_indexes)
# Update meta.json with version payloads
ocr_model = meta.get("ocr_model", meta.get("ocr_provider", "PaddleOCR"))
version_payload = build_version_payload(

View file

@ -0,0 +1,71 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from paperforge.core.io import write_json
def build_role_indexes(
structured_blocks: list[dict[str, Any]],
resolved_metadata: dict[str, Any],
) -> dict[str, list[dict[str, Any]]]:
body_roles = {
"body_paragraph",
"section_heading",
"subsection_heading",
"abstract_body",
"abstract_heading",
"introduction_heading",
}
body: list[dict[str, Any]] = []
captions: list[dict[str, Any]] = []
tables: list[dict[str, Any]] = []
references: list[dict[str, Any]] = []
all_blocks: list[dict[str, Any]] = []
for block in structured_blocks:
role = block.get("role", "")
entry = {
"paper_id": block.get("paper_id", ""),
"page": block.get("page", 0),
"block_id": block.get("block_id", ""),
"role": role,
"text": block.get("text", ""),
}
all_blocks.append(entry)
if role in body_roles:
body.append(entry)
elif role == "figure_caption":
captions.append(entry)
elif role == "table_caption":
tables.append(entry)
elif role == "reference_item":
references.append(entry)
metadata_index: list[dict[str, Any]] = []
for key in ("title", "authors", "doi", "journal", "year"):
value = resolved_metadata.get(key, {}).get("value", "")
if value:
metadata_index.append({
"paper_id": structured_blocks[0].get("paper_id", "") if structured_blocks else "",
"page": 0,
"block_id": f"meta_{key}",
"role": f"metadata_{key}",
"text": str(value),
})
return {
"body": body,
"captions": captions,
"tables": tables,
"metadata": metadata_index,
"references": references,
"all_blocks": all_blocks,
}
def write_role_index(index_root: Path, indexes: dict[str, Any]) -> None:
index_root.mkdir(parents=True, exist_ok=True)
write_json(index_root / "role-index.json", indexes)

View file

@ -128,6 +128,15 @@ def run_derived_rebuild_for_keys(vault: Path, keys: list[str]) -> dict:
)
write_ocr_health(paper_root / "health", health_report)
# Rebuild role index
from paperforge.worker.ocr_index import build_role_indexes, write_role_index
role_indexes = build_role_indexes(
structured_blocks=structured,
resolved_metadata=resolved,
)
write_role_index(paper_root / "index", role_indexes)
# Update version state in meta.json
meta = read_json(artifacts.meta_json) if artifacts.meta_json.exists() else {}
meta = _apply_post_rebuild_version_flags(meta)