From 77a6e0cfec37da026905ba593ad8a5a892aa7952 Mon Sep 17 00:00:00 2001 From: LLLin000 <809867916@qq.com> Date: Mon, 6 Jul 2026 18:19:17 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20heading=E2=86=92emitted=20events,=20back?= =?UTF-8?q?matter=E2=86=92heading=20events,=20page-qualified=20keys?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Section/subsection headings now appear in BOTH heading_events and emitted_block_events (emitted_as='heading') for complete event stream - Backmatter headings (Funding/Acknowledgments/Data Availability/Conflicts) now generate heading_events → appear as structure tree sections with their own body_units - Page-qualified block IDs (p{page}:{block_id}) in tree own/subtree_block_ids and body_units block_map: eliminates ambiguous block_id collisions when same block_id appears on multiple pages in real OCR data - TC-9.2 wording fix, KEYS empty guard, global unit_id uniqueness check --- .../memory-layer-functional-test-plan.md | 38 ++++++++------- paperforge/retrieval/structure_tree.py | 6 +-- paperforge/retrieval/units.py | 13 +++-- paperforge/worker/ocr_render.py | 21 ++++++-- tests/test_layer4_structure_tree.py | 25 +++++----- tests/test_layer4_units_and_fts.py | 48 +++++++++---------- 6 files changed, 85 insertions(+), 66 deletions(-) diff --git a/docs/plans/memory-layer-functional-test-plan.md b/docs/plans/memory-layer-functional-test-plan.md index 0bc8ed74..3ebeec69 100644 --- a/docs/plans/memory-layer-functional-test-plan.md +++ b/docs/plans/memory-layer-functional-test-plan.md @@ -52,6 +52,9 @@ KEYS = sorted( and (d / "index" / "structure-tree.json").exists() and (d / "render" / "render-map.json").exists() )[:10] +if not KEYS: + print("No eligible OCR papers found") + sys.exit(1) pass_count = 0 fail_count = 0 @@ -214,6 +217,9 @@ for key in KEYS: except Exception as e: check(f"{key}: no crash", False, str(e)) +# Global unit_id uniqueness +check("global: unit_ids unique across all papers", + len(set(all_unit_ids)) == len(all_unit_ids)) # ════════════════════════════════════════════════════════════════════════ # 4. Object Units (from real role_index) @@ -238,22 +244,22 @@ for key in KEYS: print(f" {key}: {len(object_units)} object units") - # Verify both key formats work - role_index_a = {"captions": [{"figure_id": "FigT", "text": "Test fig"}]} - role_index_b = {"figure_captions": [{"figure_id": "FigT", "text": "Test fig"}]} - empty_tree = {"paper_id": "T", "nodes": [ - {"node_id": "sec:0", "title": "T", "level": 1, "block_id": 0, - "own_block_ids": [], "subtree_block_ids": [], "children": [], - "objects": [], "page_span": [1, 1]}, - ]} - ua = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_a) - ub = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_b) - check(f"{key}: key compat (captions)", len(ua) == 1) - check(f"{key}: key compat (figure_captions)", len(ub) == 1) - except Exception as e: check(f"{key}: no crash", False, str(e)) +# Object key format compatibility (test once, not per-paper) +role_index_a = {"captions": [{"figure_id": "FigT", "text": "Test fig"}]} +role_index_b = {"figure_captions": [{"figure_id": "FigT", "text": "Test fig"}]} +empty_tree = {"paper_id": "T", "nodes": [ + {"node_id": "sec:0", "title": "T", "level": 1, "block_id": 0, + "own_block_ids": [], "subtree_block_ids": [], "children": [], + "objects": [], "page_span": [1, 1]}, +]} +ua = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_a) +ub = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_b) +check("obj key compat (captions)", len(ua) == 1) +check("obj key compat (figure_captions)", len(ub) == 1) + # ════════════════════════════════════════════════════════════════════════ # 5. FTS No Duplicate # ════════════════════════════════════════════════════════════════════════ @@ -554,11 +560,11 @@ Skip : has structured blocks but no body_units in DB. - 按 score 降序排序 ### TC-9.2: 去重 -- 同一 `(source, unit_id)` 不去重（不跨 source 去重文本） -- 同一 paper 最多 2 条（per-paper cap） +- 同一 `(source, unit_id)` 会去重； +- 不同 source 即使文本相同也不跨 source 去重； +- 最终依靠 per-paper cap 限制重复曝光。 ### TC-9.3: Per-paper cap -- 同一篇 paper 在 results 中出现 ≤ 2 次 ### TC-9.4: 空结果处理 - 返回空的 chunks diff --git a/paperforge/retrieval/structure_tree.py b/paperforge/retrieval/structure_tree.py index 7eb609bf..4fe9260b 100644 --- a/paperforge/retrieval/structure_tree.py +++ b/paperforge/retrieval/structure_tree.py @@ -42,6 +42,7 @@ def build_structure_tree( "title": h["title"], "level": h["markdown_level"], "block_id": h["block_id"], + "page": h["page"], "page_span": [h["page"], h["page"]], "own_block_ids": [], "subtree_block_ids": [], @@ -77,9 +78,8 @@ def build_structure_tree( start = bounds["start"] end = bounds["end"] - node["subtree_block_ids"] = [ - str(e["block_id"]) for e in emitted_block_events if start <= e["emitted_order"] < end + f"p{e['page']}:{str(e['block_id'])}" for e in emitted_block_events if start <= e["emitted_order"] < end ] # Extend page_span from emitted blocks @@ -99,7 +99,7 @@ def build_structure_tree( child_ids: set[str] = set() for child in node.get("children", []): child_ids.update(child.get("subtree_block_ids", [])) - child_ids.add(str(node["block_id"])) + child_ids.add(f"p{node['page']}:{str(node['block_id'])}") node["own_block_ids"] = [bid for bid in node["subtree_block_ids"] if bid not in child_ids] diff --git a/paperforge/retrieval/units.py b/paperforge/retrieval/units.py index 956af0a2..094b7f19 100644 --- a/paperforge/retrieval/units.py +++ b/paperforge/retrieval/units.py @@ -118,18 +118,17 @@ def build_body_units( if not paper_id or not tree.get("nodes"): return units - block_map: dict[str | int, dict[str, Any]] = {} + block_map: dict[str, dict[str, Any]] = {} for b in structured_blocks: bid = b.get("block_id") - if bid is not None: - key = str(bid) - # ponytail: same block_id appears on multiple pages in real OCR data - # (e.g. running headers like \"Materials\" on pages 1-7). - # Prefer the entry that has non-empty text content. + page = b.get("page") + if bid is not None and page is not None: + key = f"p{page}:{bid}" + # page-qualified key to avoid collisions when same block_id + # appears on multiple pages (common in real OCR data) existing = block_map.get(key) if existing is None or (b.get("text") and not existing.get("text")): block_map[key] = b - block_map[bid] = b def walk(node: dict[str, Any], inherited_path: list[str]) -> None: this_path = inherited_path + [node["title"]] diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index 138833ae..4a46225c 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -1821,14 +1821,20 @@ def render_fulltext_markdown( if role == "backmatter_boundary_heading" or role == "backmatter_heading": last_structured_insert_page = None last_structured_insert_bbox = None - bm_start_line = len(lines) if text and not _should_suppress_frontmatter_heading(text): lines.append(f"**{text}**") lines.append("") - if len(lines) > bm_start_line: + heading_events.append({ + "line_number": len(lines) - 1, + "markdown_level": 2, + "title": text, + "page": block_page, + "block_id": block.get("block_id"), + "emitted_order": emitted_order_counter, + }) emitted_block_events.append({ "emitted_order": emitted_order_counter, - "line_start": bm_start_line, + "line_start": len(lines) - 2, "line_end": len(lines), "page": block_page, "block_id": block.get("block_id"), @@ -1890,6 +1896,15 @@ def render_fulltext_markdown( "emitted_order": emitted_order_counter, }) emitted_order_counter += 1 + emitted_block_events.append({ + "emitted_order": emitted_order_counter - 1, + "line_start": len(lines) - 2, + "line_end": len(lines), + "page": block.get("page"), + "block_id": block.get("block_id"), + "role": role, + "emitted_as": "heading", + }) elif role == "structured_insert": si_start_line = len(lines) container_text = block.get("_container_text") diff --git a/tests/test_layer4_structure_tree.py b/tests/test_layer4_structure_tree.py index 9b9b13c7..1e4d1e72 100644 --- a/tests/test_layer4_structure_tree.py +++ b/tests/test_layer4_structure_tree.py @@ -43,8 +43,8 @@ def test_single_heading_with_body(): assert n["title"] == "Methods" assert n["level"] == 2 assert n["block_id"] == "b1" - assert set(n["subtree_block_ids"]) == {"b1", "b2"} - assert n["own_block_ids"] == ["b2"] + assert set(n["subtree_block_ids"]) == {"p1:b1", "p1:b2"} + assert n["own_block_ids"] == ["p1:b2"] def test_h2_h3_nesting(): @@ -60,8 +60,8 @@ def test_h2_h3_nesting(): child = parent["children"][0] assert parent["title"] == "Methods" assert child["title"] == "Statistics" - assert parent["own_block_ids"] == ["b2"] - assert child["own_block_ids"] == ["b4"] + assert parent["own_block_ids"] == ["p1:b2"] + assert child["own_block_ids"] == ["p1:b4"] def test_h2_h3_h2_sibling(): @@ -128,10 +128,9 @@ def test_own_block_ids_excludes_children(): tree = build_structure_tree(heading_events, emitted, structured) parent = tree["nodes"][0] child = parent["children"][0] - assert parent["own_block_ids"] == ["d2"] - assert child["own_block_ids"] == ["d4"] - assert parent["subtree_block_ids"] == ["d1", "d2", "d3", "d4"] - assert child["subtree_block_ids"] == ["d3", "d4"] + assert parent["own_block_ids"] == ["p1:d2"] + assert parent["subtree_block_ids"] == ["p1:d1", "p1:d2", "p1:d3", "p1:d4"] + assert child["subtree_block_ids"] == ["p1:d3", "p1:d4"] def test_summary_in_last_child_scope(): @@ -148,8 +147,8 @@ def test_summary_in_last_child_scope(): tree = build_structure_tree(heading_events, emitted, structured) parent = tree["nodes"][0] child = parent["children"][0] - assert parent["own_block_ids"] == ["d2"] - assert child["own_block_ids"] == ["d4", "d5"] + assert parent["own_block_ids"] == ["p1:d2"] + assert child["own_block_ids"] == ["p1:d4", "p1:d5"] def test_rendered_order_differs_from_structured_order(): @@ -157,6 +156,6 @@ def test_rendered_order_differs_from_structured_order(): emitted = [_e(0, "b3", page=1), _e(1, "b1", page=1), _e(2, "b2", page=1)] structured = [{"paper_id": "P006"}] tree = build_structure_tree(heading_events, emitted, structured) - assert "b3" not in tree["nodes"][0]["subtree_block_ids"] - assert "b1" in tree["nodes"][0]["subtree_block_ids"] - assert "b2" in tree["nodes"][0]["subtree_block_ids"] + assert "p1:b3" not in tree["nodes"][0]["subtree_block_ids"] + assert "p1:b1" in tree["nodes"][0]["subtree_block_ids"] + assert "p1:b2" in tree["nodes"][0]["subtree_block_ids"] diff --git a/tests/test_layer4_units_and_fts.py b/tests/test_layer4_units_and_fts.py index f64c0e8a..fa61cc33 100644 --- a/tests/test_layer4_units_and_fts.py +++ b/tests/test_layer4_units_and_fts.py @@ -40,12 +40,12 @@ def test_build_body_units_assigns_stable_ids_and_audit_fields(): tree = { "paper_id": "ABCD1234", "nodes": [ - _node("sec:b1", "Methods", own_block_ids=["b2"]), + _node("sec:b1", "Methods", own_block_ids=["p1:b2"]), ], } blocks = [ - {"block_id": "b1", "role": "section_heading", "text": "Methods"}, - {"block_id": "b2", "role": "body_paragraph", "text": "We recruited 30 patients."}, + {"block_id": "b1", "role": "section_heading", "text": "Methods", "page": 1}, + {"block_id": "b2", "role": "body_paragraph", "text": "We recruited 30 patients.", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert len(units) == 1 @@ -63,13 +63,13 @@ def test_build_body_units_concatenates_multiple_paragraphs(): tree = { "paper_id": "P001", "nodes": [ - _node("sec:x", "Results", own_block_ids=["b2", "b3"]), + _node("sec:x", "Results", own_block_ids=["p1:b2", "p1:b3"]), ], } blocks = [ - {"block_id": "b1", "role": "section_heading", "text": "Results"}, - {"block_id": "b2", "role": "body_paragraph", "text": "First result."}, - {"block_id": "b3", "role": "body_paragraph", "text": "Second result."}, + {"block_id": "b1", "role": "section_heading", "text": "Results", "page": 1}, + {"block_id": "b2", "role": "body_paragraph", "text": "First result.", "page": 1}, + {"block_id": "b3", "role": "body_paragraph", "text": "Second result.", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert "First result." in units[0]["unit_text"] @@ -84,7 +84,7 @@ def test_build_body_units_marks_empty_as_non_indexable(): ], } blocks = [ - {"block_id": "e1", "role": "section_heading", "text": "Empty"}, + {"block_id": "e1", "role": "section_heading", "text": "Empty", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert units == [] # no own_block_ids with body role → no units @@ -94,11 +94,11 @@ def test_build_body_units_token_estimate(): tree = { "paper_id": "P003", "nodes": [ - _node("sec:t1", "Introduction", own_block_ids=["b1"]), + _node("sec:t1", "Introduction", own_block_ids=["p1:b1"]), ], } blocks = [ - {"block_id": "b1", "role": "body_paragraph", "text": "A" * 100}, + {"block_id": "b1", "role": "body_paragraph", "text": "A" * 100, "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert len(units) == 1 @@ -114,12 +114,12 @@ def test_body_unit_excludes_reference_item(): tree = { "paper_id": "P005", "nodes": [ - _node("sec:body", "Discussion", own_block_ids=["b1", "r1"]), + _node("sec:body", "Discussion", own_block_ids=["p1:b1", "p1:r1"]), ], } blocks = [ - {"block_id": "b1", "role": "body_paragraph", "text": "Main text."}, - {"block_id": "r1", "role": "reference_item", "text": "[1] Smith et al."}, + {"block_id": "b1", "role": "body_paragraph", "text": "Main text.", "page": 1}, + {"block_id": "r1", "role": "reference_item", "text": "[1] Smith et al.", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert len(units) == 1 @@ -131,11 +131,11 @@ def test_backmatter_body_creates_separate_unit(): tree = { "paper_id": "P006", "nodes": [ - _node("sec:funding", "Funding", own_block_ids=["s1"]), + _node("sec:funding", "Funding", own_block_ids=["p1:s1"]), ], } blocks = [ - {"block_id": "s1", "role": "structured_insert", "text": "This work was funded by NIH."}, + {"block_id": "s1", "role": "structured_insert", "text": "This work was funded by NIH.", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert len(units) == 1 @@ -146,12 +146,12 @@ def test_mixed_body_and_backmatter_split(): tree = { "paper_id": "P007", "nodes": [ - _node("sec:end", "End", own_block_ids=["b1", "s1"]), + _node("sec:end", "End", own_block_ids=["p1:b1", "p1:s1"]), ], } blocks = [ - {"block_id": "b1", "role": "body_paragraph", "text": "Main text."}, - {"block_id": "s1", "role": "structured_insert", "text": "Data available on request."}, + {"block_id": "b1", "role": "body_paragraph", "text": "Main text.", "page": 1}, + {"block_id": "s1", "role": "structured_insert", "text": "Data available on request.", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) kinds = {u["unit_kind"] for u in units} @@ -165,11 +165,11 @@ def test_token_cap_splits_into_parts(): tree = { "paper_id": "P008", "nodes": [ - _node("sec:big", "Long Section", own_block_ids=["b1"]), + _node("sec:big", "Long Section", own_block_ids=["p1:b1"]), ], } blocks = [ - {"block_id": "b1", "role": "body_paragraph", "text": "word " * 1000}, + {"block_id": "b1", "role": "body_paragraph", "text": "word " * 1000, "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) assert len(units) >= 2 # should be split @@ -182,14 +182,14 @@ def test_recursive_walk_produces_correct_section_path(): tree = { "paper_id": "P009", "nodes": [ - _node("sec:root", "Root", own_block_ids=["b1"], children=[ - _node("sec:child", "Child", level=3, own_block_ids=["b2"]), + _node("sec:root", "Root", own_block_ids=["p1:b1"], children=[ + _node("sec:child", "Child", level=3, own_block_ids=["p1:b2"]), ]), ], } blocks = [ - {"block_id": "b1", "role": "body_paragraph", "text": "Root text."}, - {"block_id": "b2", "role": "body_paragraph", "text": "Child text."}, + {"block_id": "b1", "role": "body_paragraph", "text": "Root text.", "page": 1}, + {"block_id": "b2", "role": "body_paragraph", "text": "Child text.", "page": 1}, ] units = build_body_units(tree=tree, structured_blocks=blocks) paths = {u["unit_id"]: u["section_path"] for u in units}