diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index f33d7c4f..7a9697da 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -48,6 +48,7 @@ from paperforge.worker._utils import ( ) from paperforge.worker.asset_index import refresh_index_entry from paperforge.worker.ocr_artifacts import artifact_paths_for_key, build_version_payload, compute_pdf_fingerprint, compute_json_hash +from paperforge.worker.ocr_blocks import build_raw_blocks_for_result_lines, write_raw_blocks_jsonl from paperforge.worker.sync import ( load_control_actions, load_export_rows, @@ -1726,11 +1727,11 @@ def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tu } write_json(artifacts.source_metadata, source_meta) - # Ensure canonical and structure directories exist (Phase 1 placeholders) + # Emit canonical raw blocks artifacts.blocks_raw.parent.mkdir(parents=True, exist_ok=True) artifacts.blocks_structured.parent.mkdir(parents=True, exist_ok=True) - if not artifacts.blocks_raw.exists(): - artifacts.blocks_raw.write_text("", encoding="utf-8") + all_raw_blocks = build_raw_blocks_for_result_lines(key, all_results) + write_raw_blocks_jsonl(artifacts.blocks_raw, all_raw_blocks) if not artifacts.blocks_structured.exists(): artifacts.blocks_structured.write_text("", encoding="utf-8") diff --git a/paperforge/worker/ocr_blocks.py b/paperforge/worker/ocr_blocks.py new file mode 100644 index 00000000..44d5e07b --- /dev/null +++ b/paperforge/worker/ocr_blocks.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def build_raw_blocks_for_page(paper_id: str, page: int, result: dict) -> list[dict[str, Any]]: + pruned = result.get("prunedResult", {}) + width = pruned.get("width", 0) + height = pruned.get("height", 0) + blocks = pruned.get("parsing_res_list", []) + rows = [] + for i, block in enumerate(blocks): + rows.append({ + "paper_id": paper_id, + "page": page, + "block_id": block.get("block_id", f"p{page}_b{i}"), + "raw_label": block.get("block_label", "unknown"), + "raw_order": block.get("block_order", i), + "bbox": block.get("block_bbox", [0, 0, 0, 0]), + "text": block.get("block_content", "") or "", + "page_width": width, + "page_height": height, + "source": "ocr_raw", + }) + return rows + + +def build_raw_blocks_for_result_lines(paper_id: str, all_results: list[dict]) -> list[dict[str, Any]]: + rows = [] + page_num = 0 + for payload in all_results: + for res in payload.get("layoutParsingResults", []): + page_num += 1 + rows.extend(build_raw_blocks_for_page(paper_id, page_num, res)) + return rows + + +def write_raw_blocks_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") diff --git a/tests/test_ocr_blocks.py b/tests/test_ocr_blocks.py new file mode 100644 index 00000000..6f8829a3 --- /dev/null +++ b/tests/test_ocr_blocks.py @@ -0,0 +1,22 @@ +from __future__ import annotations + + +def test_build_raw_blocks_preserves_every_block() -> None: + from paperforge.worker.ocr_blocks import build_raw_blocks_for_page + + result = { + "prunedResult": { + "width": 1200, + "height": 1600, + "parsing_res_list": [ + {"block_id": 1, "block_label": "text", "block_order": 0, "block_bbox": [1, 2, 3, 4], "block_content": "A"}, + {"block_id": 2, "block_label": "header", "block_order": 1, "block_bbox": [5, 6, 7, 8], "block_content": "B"}, + ], + } + } + + rows = build_raw_blocks_for_page("KEY001", 1, result) + + assert len(rows) == 2 + assert rows[0]["paper_id"] == "KEY001" + assert rows[1]["raw_label"] == "header" diff --git a/tests/test_ocr_integration_fixtures.py b/tests/test_ocr_integration_fixtures.py index 6b6fed81..90b62526 100644 --- a/tests/test_ocr_integration_fixtures.py +++ b/tests/test_ocr_integration_fixtures.py @@ -3,6 +3,8 @@ from __future__ import annotations import json from pathlib import Path +import pytest + OCR_CORPUS = Path("D:/L/OB/Literature-hub/System/PaperForge/ocr") @@ -111,3 +113,24 @@ def test_session_regression_figure_not_in_unrelated_section() -> None: continue assert True + + +def test_fixture_raw_blocks_nonzero(tmp_path) -> None: + """Load a real result.json, write blocks.raw.jsonl, verify rows.""" + key = "2GN9LMCW" + json_path = _load_json_path(key) + if not json_path or not json_path.exists(): + pytest.skip("fixture not available") + + data = json.loads(json_path.read_text(encoding="utf-8")) + from paperforge.worker.ocr_blocks import build_raw_blocks_for_result_lines, write_raw_blocks_jsonl + + rows = build_raw_blocks_for_result_lines(key, data) + assert len(rows) > 0 + + out_dir = tmp_path / key + out_dir.mkdir(parents=True) + out_path = out_dir / "blocks.raw.jsonl" + write_raw_blocks_jsonl(out_path, rows) + assert out_path.exists() + assert out_path.stat().st_size > 0