mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
C0: add OCR_PIPELINE_V3 toggle, seed-only build_structured_blocks, stub modules
- Add _ocr_pipeline_v3_enabled() toggle function in ocr.py - Add normalize_mode='legacy'|'seed_only' param to build_structured_blocks() - Add seed_only guard that skips legacy normalize pipeline - Create ocr_pre_match_normalize.py stub - Create ocr_post_match_normalize.py stub - Create test_ocr_pipeline_v3.py with 3 tests
This commit is contained in:
parent
7bf652c42f
commit
c9d78fca44
5 changed files with 100 additions and 0 deletions
|
|
@ -101,6 +101,12 @@ OCR_QUEUE_STATUSES = {
|
|||
}
|
||||
OCR_SETTLED_STATUSES = {"done", "done_degraded", "fatal_error", "blocked"}
|
||||
|
||||
|
||||
|
||||
def _ocr_pipeline_v3_enabled() -> bool:
|
||||
import os
|
||||
|
||||
return os.environ.get("OCR_PIPELINE_V3", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
_LEGACY_HARD_DEGRADE_PREFIXES = (
|
||||
"rendered text gaps",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -190,9 +190,17 @@ def build_structured_blocks(
|
|||
raw_blocks: list[dict],
|
||||
source_metadata: dict | None = None,
|
||||
structure_output_dir: str | Path | None = None,
|
||||
normalize_mode: str = "legacy",
|
||||
) -> tuple[list[dict], Any]:
|
||||
"""Build structured blocks with role assignment, normalization, and rescue.
|
||||
|
||||
Args:
|
||||
raw_blocks: OCR raw blocks.
|
||||
source_metadata: Optional source metadata for frontmatter anchors.
|
||||
structure_output_dir: Optional output dir for document structure JSON.
|
||||
normalize_mode: "legacy" (default) runs full normalize pipeline;
|
||||
"seed_only" returns rows with seed_role as role before legacy normalize.
|
||||
|
||||
Returns (rows, doc_structure_or_None) so callers can pass the
|
||||
document structure artifact downstream without re-computing it.
|
||||
"""
|
||||
|
|
@ -316,6 +324,12 @@ def build_structured_blocks(
|
|||
)
|
||||
doc_structure.source_frontmatter_anchors = _sfm_anchors
|
||||
|
||||
# Early return for v3 seed-only mode — skip legacy normalize pipeline.
|
||||
if normalize_mode == "seed_only":
|
||||
for row in rows:
|
||||
row["role"] = row["seed_role"]
|
||||
return rows, doc_structure
|
||||
|
||||
# Normalize document structure (backmatter boundary, role regime, tail promotion)
|
||||
from paperforge.worker.ocr_document import normalize_document_structure
|
||||
|
||||
|
|
|
|||
20
paperforge/worker/ocr_post_match_normalize.py
Normal file
20
paperforge/worker/ocr_post_match_normalize.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
"""OCR post-match normalization — v3 final role commit.
|
||||
|
||||
Consumes figure/table inventories and commits the final public `role`.
|
||||
Reuses tail settlement from Workstream B.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from paperforge.worker.ocr_document import DocumentStructure
|
||||
|
||||
|
||||
def post_match_normalize(
|
||||
rows: list[dict],
|
||||
figure_inventory: dict,
|
||||
table_inventory: dict,
|
||||
*,
|
||||
document_structure: DocumentStructure,
|
||||
source_frontmatter_anchors: dict | None = None,
|
||||
) -> tuple[list[dict], DocumentStructure]:
|
||||
return rows, document_structure
|
||||
18
paperforge/worker/ocr_pre_match_normalize.py
Normal file
18
paperforge/worker/ocr_pre_match_normalize.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
"""OCR pre-match normalization — v3 candidate-only normalize.
|
||||
|
||||
Preserves public `role = seed_role`, writes `role_candidate` from a shadow
|
||||
normalize pass so figure/table matching can work from candidates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from paperforge.worker.ocr_document import DocumentStructure
|
||||
|
||||
|
||||
def pre_match_normalize(
|
||||
rows: list[dict],
|
||||
*,
|
||||
source_frontmatter_anchors: dict | None = None,
|
||||
document_structure: DocumentStructure | None = None,
|
||||
) -> tuple[list[dict], DocumentStructure]:
|
||||
return rows, document_structure or DocumentStructure()
|
||||
42
tests/test_ocr_pipeline_v3.py
Normal file
42
tests/test_ocr_pipeline_v3.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_ocr_pipeline_v3_enabled_defaults_false(monkeypatch) -> None:
|
||||
from paperforge.worker.ocr import _ocr_pipeline_v3_enabled
|
||||
|
||||
monkeypatch.delenv("OCR_PIPELINE_V3", raising=False)
|
||||
|
||||
assert _ocr_pipeline_v3_enabled() is False
|
||||
|
||||
|
||||
def test_ocr_pipeline_v3_enabled_truthy(monkeypatch) -> None:
|
||||
from paperforge.worker.ocr import _ocr_pipeline_v3_enabled
|
||||
|
||||
monkeypatch.setenv("OCR_PIPELINE_V3", "1")
|
||||
|
||||
assert _ocr_pipeline_v3_enabled() is True
|
||||
|
||||
|
||||
def test_build_structured_blocks_seed_only_skips_legacy_normalize(monkeypatch) -> None:
|
||||
import paperforge.worker.ocr_document as ocr_document
|
||||
from paperforge.worker.ocr_blocks import build_structured_blocks
|
||||
|
||||
raw_blocks = [
|
||||
{
|
||||
"paper_id": "test_paper",
|
||||
"block_id": "r1",
|
||||
"page": 1,
|
||||
"raw_label": "text",
|
||||
"raw_order": 0,
|
||||
"text": "Minimal body text.",
|
||||
"bbox": [100, 100, 420, 140],
|
||||
"page_width": 612,
|
||||
"page_height": 792,
|
||||
}
|
||||
]
|
||||
|
||||
rows, doc = build_structured_blocks(raw_blocks, normalize_mode="seed_only")
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["role"] == rows[0]["seed_role"]
|
||||
assert doc is not None
|
||||
Loading…
Reference in a new issue