From 01461330bbe65edadc927d06e31c33bc06e7ff9c Mon Sep 17 00:00:00 2001 From: LLLin000 <809867916@qq.com> Date: Mon, 6 Jul 2026 19:08:54 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20final=20hardening=20=E2=80=94=20policy?= =?UTF-8?q?=20revision=20+=20Layer=20A=20all=20green?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _body_unit_role_kind now excludes only reference_item/reference_heading (per policy: everything in fulltext except ref zone) - backmatter_body unit_kind removed; all units are 'body' - bounds_map in structure_tree uses page-qualified key (fixes collision when same block_id appears on different pages) - Layer A test plan: emitted_ids uses page-qualified keys + handles block_id=0 (falsy value) - Layer A: 1609 passed, 0 failed on real vault (22 papers) - Unit tests: 152 passed, 1 pre-existing skip --- %TEMP%test_import.py | 1 + docs/plans/memory-layer-functional-test-plan.md | 2 +- paperforge/retrieval/structure_tree.py | 4 ++-- paperforge/retrieval/units.py | 12 ++++++------ paperforge/worker/ocr_versions.py | 2 +- tests/test_layer4_units_and_fts.py | 8 +++----- 6 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 %TEMP%test_import.py diff --git a/%TEMP%test_import.py b/%TEMP%test_import.py new file mode 100644 index 00000000..94dc0568 --- /dev/null +++ b/%TEMP%test_import.py @@ -0,0 +1 @@ +import paperforge diff --git a/docs/plans/memory-layer-functional-test-plan.md b/docs/plans/memory-layer-functional-test-plan.md index 3ebeec69..b8b8a669 100644 --- a/docs/plans/memory-layer-functional-test-plan.md +++ b/docs/plans/memory-layer-functional-test-plan.md @@ -113,7 +113,7 @@ for key in KEYS: tree = read_json(OCR_ROOT / key / "index" / "structure-tree.json") nodes = all_nodes(tree) render_map = read_json(OCR_ROOT / key / "render" / "render-map.json") - emitted_ids = {str(e["block_id"]) for e in render_map["emitted_blocks"] if e.get("block_id")} + emitted_ids = {f"p{e["page"]}:{str(e["block_id"])}" for e in render_map["emitted_blocks"] if e.get("page") is not None and e.get("block_id") is not None} check(f"{key}: tree has nodes", len(nodes) > 0) diff --git a/paperforge/retrieval/structure_tree.py b/paperforge/retrieval/structure_tree.py index 4fe9260b..f50015ab 100644 --- a/paperforge/retrieval/structure_tree.py +++ b/paperforge/retrieval/structure_tree.py @@ -65,14 +65,14 @@ def build_structure_tree( if h2["markdown_level"] <= h["markdown_level"]: next_sibling = h2["emitted_order"] break - bounds_map[h["block_id"]] = { + bounds_map[f"p{h['page']}:{h['block_id']}"] = { "start": h["emitted_order"], "end": next_sibling or float("inf"), } # ── interval assignment (recursive) ── def _assign_intervals(node: dict[str, Any]) -> None: - bounds = bounds_map.get(node["block_id"]) + bounds = bounds_map.get(f"p{node['page']}:{node['block_id']}") if bounds is None: return diff --git a/paperforge/retrieval/units.py b/paperforge/retrieval/units.py index 094b7f19..82ea68b3 100644 --- a/paperforge/retrieval/units.py +++ b/paperforge/retrieval/units.py @@ -15,12 +15,12 @@ from typing import Any # ── Role helper ── def _body_unit_role_kind(role: str) -> str | None: - """Map a block's role to body unit kind, or None if excluded.""" - if role == "body_paragraph": - return "body" - if role in {"structured_insert", "non_body_insert", "backmatter_body"}: - return "backmatter_body" - return None # reference_item, reference_heading, heading, caption, asset, etc. + """Map a block's role to body unit kind, or None if excluded. + Only reference-zone blocks are excluded; everything that appears + in the rendered fulltext is fair game.""" + if role in {"reference_item", "reference_heading"}: + return None + return "body" def _split_if_oversized(text: str, max_tokens: int = 1000) -> list[str]: diff --git a/paperforge/worker/ocr_versions.py b/paperforge/worker/ocr_versions.py index 59ade494..b6c4d054 100644 --- a/paperforge/worker/ocr_versions.py +++ b/paperforge/worker/ocr_versions.py @@ -9,7 +9,7 @@ EXPECTED_CANONICAL_BLOCK_VERSION = "1.0.0" EXPECTED_STRUCTURE_VERSION = "1.0.0" EXPECTED_METADATA_RESOLVER_VERSION = "1.0.0" EXPECTED_ASSET_EXTRACTOR_VERSION = "1.0.0" -EXPECTED_RENDERER_VERSION = "2.0.0" +EXPECTED_RENDERER_VERSION = "2.1.0" EXPECTED_DOCTOR_VERSION = "1.0.0" diff --git a/tests/test_layer4_units_and_fts.py b/tests/test_layer4_units_and_fts.py index fa61cc33..304260b6 100644 --- a/tests/test_layer4_units_and_fts.py +++ b/tests/test_layer4_units_and_fts.py @@ -139,7 +139,7 @@ def test_backmatter_body_creates_separate_unit(): ] units = build_body_units(tree=tree, structured_blocks=blocks) assert len(units) == 1 - assert units[0]["unit_kind"] == "backmatter_body" + assert units[0]["unit_kind"] == "body" def test_mixed_body_and_backmatter_split(): @@ -155,10 +155,8 @@ def test_mixed_body_and_backmatter_split(): ] units = build_body_units(tree=tree, structured_blocks=blocks) kinds = {u["unit_kind"] for u in units} - assert "body" in kinds - assert "backmatter_body" in kinds - assert len(units) == 2 - assert len({u["unit_id"] for u in units}) == len(units) + assert kinds == {"body"} + assert len(units) == 1 # merged since same kind now def test_token_cap_splits_into_parts():