mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
feat: add OCR role-based search index
This commit is contained in:
parent
4f6b250ca3
commit
5d243c8e72
3 changed files with 89 additions and 0 deletions
|
|
@ -1818,6 +1818,15 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu
|
|||
write_ocr_health(ocr_root / "health", health_report)
|
||||
meta["ocr_health_overall"] = health_report["overall"]
|
||||
|
||||
# --- Phase 5: role-based OCR index ---
|
||||
from paperforge.worker.ocr_index import build_role_indexes, write_role_index
|
||||
|
||||
role_indexes = build_role_indexes(
|
||||
structured_blocks=structured,
|
||||
resolved_metadata=resolved,
|
||||
)
|
||||
write_role_index(ocr_root / "index", role_indexes)
|
||||
|
||||
# Update meta.json with version payloads
|
||||
ocr_model = meta.get("ocr_model", meta.get("ocr_provider", "PaddleOCR"))
|
||||
version_payload = build_version_payload(
|
||||
|
|
|
|||
71
paperforge/worker/ocr_index.py
Normal file
71
paperforge/worker/ocr_index.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from paperforge.core.io import write_json
|
||||
|
||||
|
||||
def build_role_indexes(
|
||||
structured_blocks: list[dict[str, Any]],
|
||||
resolved_metadata: dict[str, Any],
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
body_roles = {
|
||||
"body_paragraph",
|
||||
"section_heading",
|
||||
"subsection_heading",
|
||||
"abstract_body",
|
||||
"abstract_heading",
|
||||
"introduction_heading",
|
||||
}
|
||||
body: list[dict[str, Any]] = []
|
||||
captions: list[dict[str, Any]] = []
|
||||
tables: list[dict[str, Any]] = []
|
||||
references: list[dict[str, Any]] = []
|
||||
all_blocks: list[dict[str, Any]] = []
|
||||
|
||||
for block in structured_blocks:
|
||||
role = block.get("role", "")
|
||||
entry = {
|
||||
"paper_id": block.get("paper_id", ""),
|
||||
"page": block.get("page", 0),
|
||||
"block_id": block.get("block_id", ""),
|
||||
"role": role,
|
||||
"text": block.get("text", ""),
|
||||
}
|
||||
all_blocks.append(entry)
|
||||
|
||||
if role in body_roles:
|
||||
body.append(entry)
|
||||
elif role == "figure_caption":
|
||||
captions.append(entry)
|
||||
elif role == "table_caption":
|
||||
tables.append(entry)
|
||||
elif role == "reference_item":
|
||||
references.append(entry)
|
||||
|
||||
metadata_index: list[dict[str, Any]] = []
|
||||
for key in ("title", "authors", "doi", "journal", "year"):
|
||||
value = resolved_metadata.get(key, {}).get("value", "")
|
||||
if value:
|
||||
metadata_index.append({
|
||||
"paper_id": structured_blocks[0].get("paper_id", "") if structured_blocks else "",
|
||||
"page": 0,
|
||||
"block_id": f"meta_{key}",
|
||||
"role": f"metadata_{key}",
|
||||
"text": str(value),
|
||||
})
|
||||
|
||||
return {
|
||||
"body": body,
|
||||
"captions": captions,
|
||||
"tables": tables,
|
||||
"metadata": metadata_index,
|
||||
"references": references,
|
||||
"all_blocks": all_blocks,
|
||||
}
|
||||
|
||||
|
||||
def write_role_index(index_root: Path, indexes: dict[str, Any]) -> None:
|
||||
index_root.mkdir(parents=True, exist_ok=True)
|
||||
write_json(index_root / "role-index.json", indexes)
|
||||
|
|
@ -128,6 +128,15 @@ def run_derived_rebuild_for_keys(vault: Path, keys: list[str]) -> dict:
|
|||
)
|
||||
write_ocr_health(paper_root / "health", health_report)
|
||||
|
||||
# Rebuild role index
|
||||
from paperforge.worker.ocr_index import build_role_indexes, write_role_index
|
||||
|
||||
role_indexes = build_role_indexes(
|
||||
structured_blocks=structured,
|
||||
resolved_metadata=resolved,
|
||||
)
|
||||
write_role_index(paper_root / "index", role_indexes)
|
||||
|
||||
# Update version state in meta.json
|
||||
meta = read_json(artifacts.meta_json) if artifacts.meta_json.exists() else {}
|
||||
meta = _apply_post_rebuild_version_flags(meta)
|
||||
|
|
|
|||
Loading…
Reference in a new issue