mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: heading→emitted events, backmatter→heading events, page-qualified keys
- Section/subsection headings now appear in BOTH heading_events and
emitted_block_events (emitted_as='heading') for complete event stream
- Backmatter headings (Funding/Acknowledgments/Data Availability/Conflicts)
now generate heading_events → appear as structure tree sections with
their own body_units
- Page-qualified block IDs (p{page}:{block_id}) in tree own/subtree_block_ids
and body_units block_map: eliminates ambiguous block_id collisions when
same block_id appears on multiple pages in real OCR data
- TC-9.2 wording fix, KEYS empty guard, global unit_id uniqueness check
This commit is contained in:
LLLin000
parent
897afdaecb
commit
77a6e0cfec
6 changed files with 85 additions and 66 deletions
|
|
@ -52,6 +52,9 @@ KEYS = sorted(
|
|||
and (d / "index" / "structure-tree.json").exists()
|
||||
and (d / "render" / "render-map.json").exists()
|
||||
)[:10]
|
||||
if not KEYS:
|
||||
print("No eligible OCR papers found")
|
||||
sys.exit(1)
|
||||
|
||||
pass_count = 0
|
||||
fail_count = 0
|
||||
|
|
@ -214,6 +217,9 @@ for key in KEYS:
|
|||
|
||||
except Exception as e:
|
||||
check(f"{key}: no crash", False, str(e))
|
||||
# Global unit_id uniqueness
|
||||
check("global: unit_ids unique across all papers",
|
||||
len(set(all_unit_ids)) == len(all_unit_ids))
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════
|
||||
# 4. Object Units (from real role_index)
|
||||
|
|
@ -238,22 +244,22 @@ for key in KEYS:
|
|||
|
||||
print(f" {key}: {len(object_units)} object units")
|
||||
|
||||
# Verify both key formats work
|
||||
role_index_a = {"captions": [{"figure_id": "FigT", "text": "Test fig"}]}
|
||||
role_index_b = {"figure_captions": [{"figure_id": "FigT", "text": "Test fig"}]}
|
||||
empty_tree = {"paper_id": "T", "nodes": [
|
||||
{"node_id": "sec:0", "title": "T", "level": 1, "block_id": 0,
|
||||
"own_block_ids": [], "subtree_block_ids": [], "children": [],
|
||||
"objects": [], "page_span": [1, 1]},
|
||||
]}
|
||||
ua = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_a)
|
||||
ub = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_b)
|
||||
check(f"{key}: key compat (captions)", len(ua) == 1)
|
||||
check(f"{key}: key compat (figure_captions)", len(ub) == 1)
|
||||
|
||||
except Exception as e:
|
||||
check(f"{key}: no crash", False, str(e))
|
||||
|
||||
# Object key format compatibility (test once, not per-paper)
|
||||
role_index_a = {"captions": [{"figure_id": "FigT", "text": "Test fig"}]}
|
||||
role_index_b = {"figure_captions": [{"figure_id": "FigT", "text": "Test fig"}]}
|
||||
empty_tree = {"paper_id": "T", "nodes": [
|
||||
{"node_id": "sec:0", "title": "T", "level": 1, "block_id": 0,
|
||||
"own_block_ids": [], "subtree_block_ids": [], "children": [],
|
||||
"objects": [], "page_span": [1, 1]},
|
||||
]}
|
||||
ua = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_a)
|
||||
ub = build_object_units(tree=empty_tree, structured_blocks=[], role_index=role_index_b)
|
||||
check("obj key compat (captions)", len(ua) == 1)
|
||||
check("obj key compat (figure_captions)", len(ub) == 1)
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════
|
||||
# 5. FTS No Duplicate
|
||||
# ════════════════════════════════════════════════════════════════════════
|
||||
|
|
@ -554,11 +560,11 @@ Skip <KEY>: has structured blocks but no body_units in DB.
|
|||
- 按 score 降序排序
|
||||
|
||||
### TC-9.2: 去重
|
||||
- 同一 `(source, unit_id)` 不去重（不跨 source 去重文本）
|
||||
- 同一 paper 最多 2 条（per-paper cap）
|
||||
- 同一 `(source, unit_id)` 会去重；
|
||||
- 不同 source 即使文本相同也不跨 source 去重；
|
||||
- 最终依靠 per-paper cap 限制重复曝光。
|
||||
|
||||
### TC-9.3: Per-paper cap
|
||||
- 同一篇 paper 在 results 中出现 ≤ 2 次
|
||||
|
||||
### TC-9.4: 空结果处理
|
||||
- 返回空的 chunks
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ def build_structure_tree(
|
|||
"title": h["title"],
|
||||
"level": h["markdown_level"],
|
||||
"block_id": h["block_id"],
|
||||
"page": h["page"],
|
||||
"page_span": [h["page"], h["page"]],
|
||||
"own_block_ids": [],
|
||||
"subtree_block_ids": [],
|
||||
|
|
@ -77,9 +78,8 @@ def build_structure_tree(
|
|||
|
||||
start = bounds["start"]
|
||||
end = bounds["end"]
|
||||
|
||||
node["subtree_block_ids"] = [
|
||||
str(e["block_id"]) for e in emitted_block_events if start <= e["emitted_order"] < end
|
||||
f"p{e['page']}:{str(e['block_id'])}" for e in emitted_block_events if start <= e["emitted_order"] < end
|
||||
]
|
||||
|
||||
# Extend page_span from emitted blocks
|
||||
|
|
@ -99,7 +99,7 @@ def build_structure_tree(
|
|||
child_ids: set[str] = set()
|
||||
for child in node.get("children", []):
|
||||
child_ids.update(child.get("subtree_block_ids", []))
|
||||
child_ids.add(str(node["block_id"]))
|
||||
child_ids.add(f"p{node['page']}:{str(node['block_id'])}")
|
||||
|
||||
node["own_block_ids"] = [bid for bid in node["subtree_block_ids"] if bid not in child_ids]
|
||||
|
||||
|
|
|
|||
|
|
@ -118,18 +118,17 @@ def build_body_units(
|
|||
if not paper_id or not tree.get("nodes"):
|
||||
return units
|
||||
|
||||
block_map: dict[str | int, dict[str, Any]] = {}
|
||||
block_map: dict[str, dict[str, Any]] = {}
|
||||
for b in structured_blocks:
|
||||
bid = b.get("block_id")
|
||||
if bid is not None:
|
||||
key = str(bid)
|
||||
# ponytail: same block_id appears on multiple pages in real OCR data
|
||||
# (e.g. running headers like \"Materials\" on pages 1-7).
|
||||
# Prefer the entry that has non-empty text content.
|
||||
page = b.get("page")
|
||||
if bid is not None and page is not None:
|
||||
key = f"p{page}:{bid}"
|
||||
# page-qualified key to avoid collisions when same block_id
|
||||
# appears on multiple pages (common in real OCR data)
|
||||
existing = block_map.get(key)
|
||||
if existing is None or (b.get("text") and not existing.get("text")):
|
||||
block_map[key] = b
|
||||
block_map[bid] = b
|
||||
def walk(node: dict[str, Any], inherited_path: list[str]) -> None:
|
||||
this_path = inherited_path + [node["title"]]
|
||||
|
||||
|
|
|
|||
|
|
@ -1821,14 +1821,20 @@ def render_fulltext_markdown(
|
|||
if role == "backmatter_boundary_heading" or role == "backmatter_heading":
|
||||
last_structured_insert_page = None
|
||||
last_structured_insert_bbox = None
|
||||
bm_start_line = len(lines)
|
||||
if text and not _should_suppress_frontmatter_heading(text):
|
||||
lines.append(f"**{text}**")
|
||||
lines.append("")
|
||||
if len(lines) > bm_start_line:
|
||||
heading_events.append({
|
||||
"line_number": len(lines) - 1,
|
||||
"markdown_level": 2,
|
||||
"title": text,
|
||||
"page": block_page,
|
||||
"block_id": block.get("block_id"),
|
||||
"emitted_order": emitted_order_counter,
|
||||
})
|
||||
emitted_block_events.append({
|
||||
"emitted_order": emitted_order_counter,
|
||||
"line_start": bm_start_line,
|
||||
"line_start": len(lines) - 2,
|
||||
"line_end": len(lines),
|
||||
"page": block_page,
|
||||
"block_id": block.get("block_id"),
|
||||
|
|
@ -1890,6 +1896,15 @@ def render_fulltext_markdown(
|
|||
"emitted_order": emitted_order_counter,
|
||||
})
|
||||
emitted_order_counter += 1
|
||||
emitted_block_events.append({
|
||||
"emitted_order": emitted_order_counter - 1,
|
||||
"line_start": len(lines) - 2,
|
||||
"line_end": len(lines),
|
||||
"page": block.get("page"),
|
||||
"block_id": block.get("block_id"),
|
||||
"role": role,
|
||||
"emitted_as": "heading",
|
||||
})
|
||||
elif role == "structured_insert":
|
||||
si_start_line = len(lines)
|
||||
container_text = block.get("_container_text")
|
||||
|
|
|
|||
|
|
@ -43,8 +43,8 @@ def test_single_heading_with_body():
|
|||
assert n["title"] == "Methods"
|
||||
assert n["level"] == 2
|
||||
assert n["block_id"] == "b1"
|
||||
assert set(n["subtree_block_ids"]) == {"b1", "b2"}
|
||||
assert n["own_block_ids"] == ["b2"]
|
||||
assert set(n["subtree_block_ids"]) == {"p1:b1", "p1:b2"}
|
||||
assert n["own_block_ids"] == ["p1:b2"]
|
||||
|
||||
|
||||
def test_h2_h3_nesting():
|
||||
|
|
@ -60,8 +60,8 @@ def test_h2_h3_nesting():
|
|||
child = parent["children"][0]
|
||||
assert parent["title"] == "Methods"
|
||||
assert child["title"] == "Statistics"
|
||||
assert parent["own_block_ids"] == ["b2"]
|
||||
assert child["own_block_ids"] == ["b4"]
|
||||
assert parent["own_block_ids"] == ["p1:b2"]
|
||||
assert child["own_block_ids"] == ["p1:b4"]
|
||||
|
||||
|
||||
def test_h2_h3_h2_sibling():
|
||||
|
|
@ -128,10 +128,9 @@ def test_own_block_ids_excludes_children():
|
|||
tree = build_structure_tree(heading_events, emitted, structured)
|
||||
parent = tree["nodes"][0]
|
||||
child = parent["children"][0]
|
||||
assert parent["own_block_ids"] == ["d2"]
|
||||
assert child["own_block_ids"] == ["d4"]
|
||||
assert parent["subtree_block_ids"] == ["d1", "d2", "d3", "d4"]
|
||||
assert child["subtree_block_ids"] == ["d3", "d4"]
|
||||
assert parent["own_block_ids"] == ["p1:d2"]
|
||||
assert parent["subtree_block_ids"] == ["p1:d1", "p1:d2", "p1:d3", "p1:d4"]
|
||||
assert child["subtree_block_ids"] == ["p1:d3", "p1:d4"]
|
||||
|
||||
|
||||
def test_summary_in_last_child_scope():
|
||||
|
|
@ -148,8 +147,8 @@ def test_summary_in_last_child_scope():
|
|||
tree = build_structure_tree(heading_events, emitted, structured)
|
||||
parent = tree["nodes"][0]
|
||||
child = parent["children"][0]
|
||||
assert parent["own_block_ids"] == ["d2"]
|
||||
assert child["own_block_ids"] == ["d4", "d5"]
|
||||
assert parent["own_block_ids"] == ["p1:d2"]
|
||||
assert child["own_block_ids"] == ["p1:d4", "p1:d5"]
|
||||
|
||||
|
||||
def test_rendered_order_differs_from_structured_order():
|
||||
|
|
@ -157,6 +156,6 @@ def test_rendered_order_differs_from_structured_order():
|
|||
emitted = [_e(0, "b3", page=1), _e(1, "b1", page=1), _e(2, "b2", page=1)]
|
||||
structured = [{"paper_id": "P006"}]
|
||||
tree = build_structure_tree(heading_events, emitted, structured)
|
||||
assert "b3" not in tree["nodes"][0]["subtree_block_ids"]
|
||||
assert "b1" in tree["nodes"][0]["subtree_block_ids"]
|
||||
assert "b2" in tree["nodes"][0]["subtree_block_ids"]
|
||||
assert "p1:b3" not in tree["nodes"][0]["subtree_block_ids"]
|
||||
assert "p1:b1" in tree["nodes"][0]["subtree_block_ids"]
|
||||
assert "p1:b2" in tree["nodes"][0]["subtree_block_ids"]
|
||||
|
|
|
|||
|
|
@ -40,12 +40,12 @@ def test_build_body_units_assigns_stable_ids_and_audit_fields():
|
|||
tree = {
|
||||
"paper_id": "ABCD1234",
|
||||
"nodes": [
|
||||
_node("sec:b1", "Methods", own_block_ids=["b2"]),
|
||||
_node("sec:b1", "Methods", own_block_ids=["p1:b2"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "section_heading", "text": "Methods"},
|
||||
{"block_id": "b2", "role": "body_paragraph", "text": "We recruited 30 patients."},
|
||||
{"block_id": "b1", "role": "section_heading", "text": "Methods", "page": 1},
|
||||
{"block_id": "b2", "role": "body_paragraph", "text": "We recruited 30 patients.", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert len(units) == 1
|
||||
|
|
@ -63,13 +63,13 @@ def test_build_body_units_concatenates_multiple_paragraphs():
|
|||
tree = {
|
||||
"paper_id": "P001",
|
||||
"nodes": [
|
||||
_node("sec:x", "Results", own_block_ids=["b2", "b3"]),
|
||||
_node("sec:x", "Results", own_block_ids=["p1:b2", "p1:b3"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "section_heading", "text": "Results"},
|
||||
{"block_id": "b2", "role": "body_paragraph", "text": "First result."},
|
||||
{"block_id": "b3", "role": "body_paragraph", "text": "Second result."},
|
||||
{"block_id": "b1", "role": "section_heading", "text": "Results", "page": 1},
|
||||
{"block_id": "b2", "role": "body_paragraph", "text": "First result.", "page": 1},
|
||||
{"block_id": "b3", "role": "body_paragraph", "text": "Second result.", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert "First result." in units[0]["unit_text"]
|
||||
|
|
@ -84,7 +84,7 @@ def test_build_body_units_marks_empty_as_non_indexable():
|
|||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "e1", "role": "section_heading", "text": "Empty"},
|
||||
{"block_id": "e1", "role": "section_heading", "text": "Empty", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert units == [] # no own_block_ids with body role → no units
|
||||
|
|
@ -94,11 +94,11 @@ def test_build_body_units_token_estimate():
|
|||
tree = {
|
||||
"paper_id": "P003",
|
||||
"nodes": [
|
||||
_node("sec:t1", "Introduction", own_block_ids=["b1"]),
|
||||
_node("sec:t1", "Introduction", own_block_ids=["p1:b1"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "A" * 100},
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "A" * 100, "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert len(units) == 1
|
||||
|
|
@ -114,12 +114,12 @@ def test_body_unit_excludes_reference_item():
|
|||
tree = {
|
||||
"paper_id": "P005",
|
||||
"nodes": [
|
||||
_node("sec:body", "Discussion", own_block_ids=["b1", "r1"]),
|
||||
_node("sec:body", "Discussion", own_block_ids=["p1:b1", "p1:r1"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "Main text."},
|
||||
{"block_id": "r1", "role": "reference_item", "text": "[1] Smith et al."},
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "Main text.", "page": 1},
|
||||
{"block_id": "r1", "role": "reference_item", "text": "[1] Smith et al.", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert len(units) == 1
|
||||
|
|
@ -131,11 +131,11 @@ def test_backmatter_body_creates_separate_unit():
|
|||
tree = {
|
||||
"paper_id": "P006",
|
||||
"nodes": [
|
||||
_node("sec:funding", "Funding", own_block_ids=["s1"]),
|
||||
_node("sec:funding", "Funding", own_block_ids=["p1:s1"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "s1", "role": "structured_insert", "text": "This work was funded by NIH."},
|
||||
{"block_id": "s1", "role": "structured_insert", "text": "This work was funded by NIH.", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert len(units) == 1
|
||||
|
|
@ -146,12 +146,12 @@ def test_mixed_body_and_backmatter_split():
|
|||
tree = {
|
||||
"paper_id": "P007",
|
||||
"nodes": [
|
||||
_node("sec:end", "End", own_block_ids=["b1", "s1"]),
|
||||
_node("sec:end", "End", own_block_ids=["p1:b1", "p1:s1"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "Main text."},
|
||||
{"block_id": "s1", "role": "structured_insert", "text": "Data available on request."},
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "Main text.", "page": 1},
|
||||
{"block_id": "s1", "role": "structured_insert", "text": "Data available on request.", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
kinds = {u["unit_kind"] for u in units}
|
||||
|
|
@ -165,11 +165,11 @@ def test_token_cap_splits_into_parts():
|
|||
tree = {
|
||||
"paper_id": "P008",
|
||||
"nodes": [
|
||||
_node("sec:big", "Long Section", own_block_ids=["b1"]),
|
||||
_node("sec:big", "Long Section", own_block_ids=["p1:b1"]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "word " * 1000},
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "word " * 1000, "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
assert len(units) >= 2 # should be split
|
||||
|
|
@ -182,14 +182,14 @@ def test_recursive_walk_produces_correct_section_path():
|
|||
tree = {
|
||||
"paper_id": "P009",
|
||||
"nodes": [
|
||||
_node("sec:root", "Root", own_block_ids=["b1"], children=[
|
||||
_node("sec:child", "Child", level=3, own_block_ids=["b2"]),
|
||||
_node("sec:root", "Root", own_block_ids=["p1:b1"], children=[
|
||||
_node("sec:child", "Child", level=3, own_block_ids=["p1:b2"]),
|
||||
]),
|
||||
],
|
||||
}
|
||||
blocks = [
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "Root text."},
|
||||
{"block_id": "b2", "role": "body_paragraph", "text": "Child text."},
|
||||
{"block_id": "b1", "role": "body_paragraph", "text": "Root text.", "page": 1},
|
||||
{"block_id": "b2", "role": "body_paragraph", "text": "Child text.", "page": 1},
|
||||
]
|
||||
units = build_body_units(tree=tree, structured_blocks=blocks)
|
||||
paths = {u["unit_id"]: u["section_path"] for u in units}
|
||||
|
|
|
|||
Loading…
Reference in a new issue