fix: anchor OCR frontmatter to source metadata

This commit is contained in:
Research Assistant 2026-06-07 15:58:38 +08:00
parent f10d244631
commit a7d1c1176f
4 changed files with 196 additions and 1 deletions

View file

@ -129,9 +129,51 @@ def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, An
return candidates
def _token_overlap(a: str, b: str) -> float:
ta = set(re.findall(r"[a-zA-Z0-9]+", a))
tb = set(re.findall(r"[a-zA-Z0-9]+", b))
if not ta or not tb:
return 0.0
return len(ta & tb) / min(len(ta), len(tb))
def _align_frontmatter_to_source_metadata(
source_meta: dict,
page1_blocks: list[dict],
) -> dict:
result: dict[str, Any] = {}
source_title = (source_meta.get("title") or "").strip()
if source_title:
result["title"] = {"source": "zotero", "value": source_title}
for b in page1_blocks:
t = (b.get("text") or b.get("block_content") or "").strip()
if len(t) >= 20 and (
t in source_title or source_title in t
or _token_overlap(t.lower(), source_title.lower()) > 0.6
):
result["title"]["ocr_block_id"] = b.get("block_id")
result["title"]["ocr_aligned"] = True
break
source_authors = source_meta.get("authors") or []
if source_authors:
result["authors"] = {"source": "zotero", "value": source_authors}
for b in page1_blocks:
t = (b.get("text") or b.get("block_content") or "").strip()
match = _match_author_block_to_source_authors(t, source_authors)
if match.get("matched"):
result["authors"]["ocr_block_id"] = b.get("block_id")
result["authors"]["ocr_aligned"] = True
break
return result
def resolve_metadata(
source_metadata: dict[str, Any],
frontmatter_candidates: dict[str, Any] | None = None,
page1_blocks: list[dict] | None = None,
) -> dict[str, Any]:
if frontmatter_candidates is None:
frontmatter_candidates = {}
@ -225,6 +267,16 @@ def resolve_metadata(
else:
resolved["authors_display"] = ""
# --- frontmatter alignment (page-1 block anchoring) ---
if page1_blocks:
alignment = _align_frontmatter_to_source_metadata(source_metadata, page1_blocks)
if "title" in resolved and "title" in alignment and alignment["title"].get("ocr_aligned"):
resolved["title"]["ocr_block_id"] = alignment["title"]["ocr_block_id"]
resolved["title"]["ocr_aligned"] = True
if "authors" in resolved and "authors" in alignment and alignment["authors"].get("ocr_aligned"):
resolved["authors"]["ocr_block_id"] = alignment["authors"]["ocr_block_id"]
resolved["authors"]["ocr_aligned"] = True
# --- year ---
zotero_year = source_metadata.get("year", 0)
if zotero_year:

View file

@ -306,6 +306,14 @@ def _infer_heading_level(
return "section_heading"
def _seems_like_authors(text: str) -> bool:
parts = [p.strip() for p in text.replace(" ", " ").split(",")]
if len(parts) >= 2:
name_count = sum(1 for p in parts if re.search(r"[A-Z][a-z]+\s+[A-Z]", p))
return name_count >= 2
return False
def assign_block_role(
block: dict,
page_blocks: list[dict],
@ -414,6 +422,29 @@ def assign_block_role(
# zone == "abstract_zone" → let existing logic handle
# ---- Page-1 frontmatter guard ----
_pg = block.get("page", 1) or 1
if _pg == 1 and raw_label in ("doc_title", "paragraph_title", "text"):
_tv = text
_lv = _tv.lower().strip().lstrip("*•·-–—")
block_bbox_local = block.get("block_bbox", [0, 0, 0, 0])
if _lv in _BACKMATTER_TITLE_DENY_LIST:
pass # let existing paragraph_title or fallthrough handle it
elif raw_label == "doc_title" or (
raw_label != "text"
and len(block_bbox_local) >= 4 and block_bbox_local[1] < page_height * 0.25
and len(_tv) > 20
and "abstract" not in _lv[:15]
and "keywords" not in _lv[:15]
and "introduction" not in _lv[:15]
):
if len(_tv) > 20 and not _seems_like_authors(_tv):
return RoleAssignment(
role="paper_title",
confidence=0.6,
evidence=[f"page-1 frontmatter title guard: {_tv[:60]}"],
)
# Paddle priors
if raw_label == "paragraph_title":
stripped = text.strip().lstrip("*•·-–—")
@ -499,7 +530,7 @@ def assign_block_role(
)
# Author-like heading guard: prevent bylines from becoming headings
_is_author_byline = (
re.search(r"&|,.*,", text)
(re.search(r"&|,.*,", text) or _seems_like_authors(text))
and len(text.split()) <= 15
and not _has_heading_numbering(text)
and not any(text.lower().startswith(w) for w in ["abstract", "introduction", "methods", "results", "discussion", "conclusion", "references"])

View file

@ -141,3 +141,78 @@ def test_resolve_metadata_author_no_zotero_fallback() -> None:
resolved = resolve_metadata(source, candidates)
assert resolved["authors"]["source"] == "ocr_frontmatter"
assert resolved["authors"]["value"] == ["Alice Smith", "Bob Jones"]
def test_title_anchored_from_source_metadata() -> None:
from paperforge.worker.ocr_metadata import (
_align_frontmatter_to_source_metadata,
)
source_meta = {
"title": "Correct Research Title That Is Long Enough",
"authors": ["Alice Smith", "Bob Jones"],
}
page1_blocks = [
{
"block_id": "p1_b1",
"block_label": "doc_title",
"block_content": "Correct Research Title That Is Long Enough",
"block_bbox": [100, 50, 900, 100],
"page": 1,
}
]
aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks)
assert aligned["title"]["source"] == "zotero"
assert aligned["title"]["value"] == "Correct Research Title That Is Long Enough"
assert aligned["title"].get("ocr_aligned") is True
def test_author_anchored_from_source_metadata() -> None:
from paperforge.worker.ocr_metadata import (
_align_frontmatter_to_source_metadata,
)
source_meta = {
"title": "Some Title",
"authors": ["Alice Smith", "Bob Jones"],
}
page1_blocks = [
{
"block_id": "p1_b2",
"block_label": "text",
"block_content": "Alice Smith, Bob Jones",
"block_bbox": [100, 200, 800, 230],
"page": 1,
}
]
aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks)
assert aligned["authors"]["source"] == "zotero"
assert aligned["authors"]["value"] == ["Alice Smith", "Bob Jones"]
assert aligned["authors"].get("ocr_aligned") is True
def test_mismatched_ocr_title_does_not_pollute_metadata() -> None:
from paperforge.worker.ocr_metadata import (
_align_frontmatter_to_source_metadata,
)
source_meta = {
"title": "Correct Zotero Title That Is Real",
"authors": ["Alice Smith"],
}
page1_blocks = [
{
"block_id": "p1_b1",
"block_label": "doc_title",
"block_content": "Totally Wrong OCR Title That Should Not Be Used",
"block_bbox": [100, 50, 900, 100],
"page": 1,
}
]
aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks)
assert aligned["title"]["value"] == "Correct Zotero Title That Is Real"
assert aligned["title"]["source"] == "zotero"
assert aligned["title"].get("ocr_aligned", False) is False

View file

@ -533,6 +533,43 @@ def test_running_header_not_heading():
assert result.role == "noise", f"Expected noise, got {result.role}"
def test_doc_title_not_body_paragraph() -> None:
from paperforge.worker.ocr_roles import assign_block_role
block = {
"block_label": "doc_title",
"block_content": "A Very Long Title That Would Normally Be Considered A Document Title Block With Sufficient Length",
"block_bbox": [100, 50, 900, 100],
"page": 1,
}
result = assign_block_role(block, page_blocks=[block], page_width=900, page_height=1200)
assert result.role != "body_paragraph", (
f"doc_title should not be body_paragraph, got {result.role}"
)
def test_author_byline_comma_only_not_section_heading() -> None:
"""Comma-separated byline on page 1 below zone detection threshold -> NOT heading."""
from paperforge.worker.ocr_roles import assign_block_role
block = {
"block_label": "paragraph_title",
"block_content": "John Smith, Jane Doe",
"block_bbox": [100, 400, 500, 430],
"page": 1,
}
page_blocks = [block]
result = assign_block_role(
block,
page_blocks=page_blocks,
page_width=600,
page_height=800,
)
assert result.role not in ("section_heading", "subsection_heading", "sub_subsection_heading"), (
f"Comma-separated byline should not be a heading, got {result.role}"
)
def test_backmatter_boundary_detects_on_early_page() -> None:
"""Backmatter boundary should be detectable on papers with fewer
than 8 pages, without a hard page gate."""