From c9d78fca44ffae4e78db08a89695af9ba9adc36a Mon Sep 17 00:00:00 2001 From: LLLin000 <809867916@qq.com> Date: Sat, 4 Jul 2026 15:54:53 +0800 Subject: [PATCH] C0: add OCR_PIPELINE_V3 toggle, seed-only build_structured_blocks, stub modules - Add _ocr_pipeline_v3_enabled() toggle function in ocr.py - Add normalize_mode='legacy'|'seed_only' param to build_structured_blocks() - Add seed_only guard that skips legacy normalize pipeline - Create ocr_pre_match_normalize.py stub - Create ocr_post_match_normalize.py stub - Create test_ocr_pipeline_v3.py with 3 tests --- paperforge/worker/ocr.py | 6 +++ paperforge/worker/ocr_blocks.py | 14 +++++++ paperforge/worker/ocr_post_match_normalize.py | 20 +++++++++ paperforge/worker/ocr_pre_match_normalize.py | 18 ++++++++ tests/test_ocr_pipeline_v3.py | 42 +++++++++++++++++++ 5 files changed, 100 insertions(+) create mode 100644 paperforge/worker/ocr_post_match_normalize.py create mode 100644 paperforge/worker/ocr_pre_match_normalize.py create mode 100644 tests/test_ocr_pipeline_v3.py diff --git a/paperforge/worker/ocr.py b/paperforge/worker/ocr.py index 3243c323..2cf9103c 100644 --- a/paperforge/worker/ocr.py +++ b/paperforge/worker/ocr.py @@ -101,6 +101,12 @@ OCR_QUEUE_STATUSES = { } OCR_SETTLED_STATUSES = {"done", "done_degraded", "fatal_error", "blocked"} + + +def _ocr_pipeline_v3_enabled() -> bool: + import os + + return os.environ.get("OCR_PIPELINE_V3", "").strip().lower() in {"1", "true", "yes", "on"} _LEGACY_HARD_DEGRADE_PREFIXES = ( "rendered text gaps", ) diff --git a/paperforge/worker/ocr_blocks.py b/paperforge/worker/ocr_blocks.py index 7d4b1be0..0d2ed1aa 100644 --- a/paperforge/worker/ocr_blocks.py +++ b/paperforge/worker/ocr_blocks.py @@ -190,9 +190,17 @@ def build_structured_blocks( raw_blocks: list[dict], source_metadata: dict | None = None, structure_output_dir: str | Path | None = None, + normalize_mode: str = "legacy", ) -> tuple[list[dict], Any]: """Build structured blocks with role assignment, normalization, and rescue. + Args: + raw_blocks: OCR raw blocks. + source_metadata: Optional source metadata for frontmatter anchors. + structure_output_dir: Optional output dir for document structure JSON. + normalize_mode: "legacy" (default) runs full normalize pipeline; + "seed_only" returns rows with seed_role as role before legacy normalize. + Returns (rows, doc_structure_or_None) so callers can pass the document structure artifact downstream without re-computing it. """ @@ -316,6 +324,12 @@ def build_structured_blocks( ) doc_structure.source_frontmatter_anchors = _sfm_anchors + # Early return for v3 seed-only mode — skip legacy normalize pipeline. + if normalize_mode == "seed_only": + for row in rows: + row["role"] = row["seed_role"] + return rows, doc_structure + # Normalize document structure (backmatter boundary, role regime, tail promotion) from paperforge.worker.ocr_document import normalize_document_structure diff --git a/paperforge/worker/ocr_post_match_normalize.py b/paperforge/worker/ocr_post_match_normalize.py new file mode 100644 index 00000000..437b593c --- /dev/null +++ b/paperforge/worker/ocr_post_match_normalize.py @@ -0,0 +1,20 @@ +"""OCR post-match normalization — v3 final role commit. + +Consumes figure/table inventories and commits the final public `role`. +Reuses tail settlement from Workstream B. +""" + +from __future__ import annotations + +from paperforge.worker.ocr_document import DocumentStructure + + +def post_match_normalize( + rows: list[dict], + figure_inventory: dict, + table_inventory: dict, + *, + document_structure: DocumentStructure, + source_frontmatter_anchors: dict | None = None, +) -> tuple[list[dict], DocumentStructure]: + return rows, document_structure diff --git a/paperforge/worker/ocr_pre_match_normalize.py b/paperforge/worker/ocr_pre_match_normalize.py new file mode 100644 index 00000000..75a7c5f3 --- /dev/null +++ b/paperforge/worker/ocr_pre_match_normalize.py @@ -0,0 +1,18 @@ +"""OCR pre-match normalization — v3 candidate-only normalize. + +Preserves public `role = seed_role`, writes `role_candidate` from a shadow +normalize pass so figure/table matching can work from candidates. +""" + +from __future__ import annotations + +from paperforge.worker.ocr_document import DocumentStructure + + +def pre_match_normalize( + rows: list[dict], + *, + source_frontmatter_anchors: dict | None = None, + document_structure: DocumentStructure | None = None, +) -> tuple[list[dict], DocumentStructure]: + return rows, document_structure or DocumentStructure() diff --git a/tests/test_ocr_pipeline_v3.py b/tests/test_ocr_pipeline_v3.py new file mode 100644 index 00000000..eb4de19f --- /dev/null +++ b/tests/test_ocr_pipeline_v3.py @@ -0,0 +1,42 @@ +from __future__ import annotations + + +def test_ocr_pipeline_v3_enabled_defaults_false(monkeypatch) -> None: + from paperforge.worker.ocr import _ocr_pipeline_v3_enabled + + monkeypatch.delenv("OCR_PIPELINE_V3", raising=False) + + assert _ocr_pipeline_v3_enabled() is False + + +def test_ocr_pipeline_v3_enabled_truthy(monkeypatch) -> None: + from paperforge.worker.ocr import _ocr_pipeline_v3_enabled + + monkeypatch.setenv("OCR_PIPELINE_V3", "1") + + assert _ocr_pipeline_v3_enabled() is True + + +def test_build_structured_blocks_seed_only_skips_legacy_normalize(monkeypatch) -> None: + import paperforge.worker.ocr_document as ocr_document + from paperforge.worker.ocr_blocks import build_structured_blocks + + raw_blocks = [ + { + "paper_id": "test_paper", + "block_id": "r1", + "page": 1, + "raw_label": "text", + "raw_order": 0, + "text": "Minimal body text.", + "bbox": [100, 100, 420, 140], + "page_width": 612, + "page_height": 792, + } + ] + + rows, doc = build_structured_blocks(raw_blocks, normalize_mode="seed_only") + + assert len(rows) == 1 + assert rows[0]["role"] == rows[0]["seed_role"] + assert doc is not None