mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: anchor OCR frontmatter to source metadata
This commit is contained in:
parent
f10d244631
commit
a7d1c1176f
4 changed files with 196 additions and 1 deletions
|
|
@ -129,9 +129,51 @@ def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, An
|
|||
return candidates
|
||||
|
||||
|
||||
def _token_overlap(a: str, b: str) -> float:
|
||||
ta = set(re.findall(r"[a-zA-Z0-9]+", a))
|
||||
tb = set(re.findall(r"[a-zA-Z0-9]+", b))
|
||||
if not ta or not tb:
|
||||
return 0.0
|
||||
return len(ta & tb) / min(len(ta), len(tb))
|
||||
|
||||
|
||||
def _align_frontmatter_to_source_metadata(
|
||||
source_meta: dict,
|
||||
page1_blocks: list[dict],
|
||||
) -> dict:
|
||||
result: dict[str, Any] = {}
|
||||
|
||||
source_title = (source_meta.get("title") or "").strip()
|
||||
if source_title:
|
||||
result["title"] = {"source": "zotero", "value": source_title}
|
||||
for b in page1_blocks:
|
||||
t = (b.get("text") or b.get("block_content") or "").strip()
|
||||
if len(t) >= 20 and (
|
||||
t in source_title or source_title in t
|
||||
or _token_overlap(t.lower(), source_title.lower()) > 0.6
|
||||
):
|
||||
result["title"]["ocr_block_id"] = b.get("block_id")
|
||||
result["title"]["ocr_aligned"] = True
|
||||
break
|
||||
|
||||
source_authors = source_meta.get("authors") or []
|
||||
if source_authors:
|
||||
result["authors"] = {"source": "zotero", "value": source_authors}
|
||||
for b in page1_blocks:
|
||||
t = (b.get("text") or b.get("block_content") or "").strip()
|
||||
match = _match_author_block_to_source_authors(t, source_authors)
|
||||
if match.get("matched"):
|
||||
result["authors"]["ocr_block_id"] = b.get("block_id")
|
||||
result["authors"]["ocr_aligned"] = True
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def resolve_metadata(
|
||||
source_metadata: dict[str, Any],
|
||||
frontmatter_candidates: dict[str, Any] | None = None,
|
||||
page1_blocks: list[dict] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if frontmatter_candidates is None:
|
||||
frontmatter_candidates = {}
|
||||
|
|
@ -225,6 +267,16 @@ def resolve_metadata(
|
|||
else:
|
||||
resolved["authors_display"] = ""
|
||||
|
||||
# --- frontmatter alignment (page-1 block anchoring) ---
|
||||
if page1_blocks:
|
||||
alignment = _align_frontmatter_to_source_metadata(source_metadata, page1_blocks)
|
||||
if "title" in resolved and "title" in alignment and alignment["title"].get("ocr_aligned"):
|
||||
resolved["title"]["ocr_block_id"] = alignment["title"]["ocr_block_id"]
|
||||
resolved["title"]["ocr_aligned"] = True
|
||||
if "authors" in resolved and "authors" in alignment and alignment["authors"].get("ocr_aligned"):
|
||||
resolved["authors"]["ocr_block_id"] = alignment["authors"]["ocr_block_id"]
|
||||
resolved["authors"]["ocr_aligned"] = True
|
||||
|
||||
# --- year ---
|
||||
zotero_year = source_metadata.get("year", 0)
|
||||
if zotero_year:
|
||||
|
|
|
|||
|
|
@ -306,6 +306,14 @@ def _infer_heading_level(
|
|||
return "section_heading"
|
||||
|
||||
|
||||
def _seems_like_authors(text: str) -> bool:
|
||||
parts = [p.strip() for p in text.replace(" ", " ").split(",")]
|
||||
if len(parts) >= 2:
|
||||
name_count = sum(1 for p in parts if re.search(r"[A-Z][a-z]+\s+[A-Z]", p))
|
||||
return name_count >= 2
|
||||
return False
|
||||
|
||||
|
||||
def assign_block_role(
|
||||
block: dict,
|
||||
page_blocks: list[dict],
|
||||
|
|
@ -414,6 +422,29 @@ def assign_block_role(
|
|||
|
||||
# zone == "abstract_zone" → let existing logic handle
|
||||
|
||||
# ---- Page-1 frontmatter guard ----
|
||||
_pg = block.get("page", 1) or 1
|
||||
if _pg == 1 and raw_label in ("doc_title", "paragraph_title", "text"):
|
||||
_tv = text
|
||||
_lv = _tv.lower().strip().lstrip("*•·-–—")
|
||||
block_bbox_local = block.get("block_bbox", [0, 0, 0, 0])
|
||||
if _lv in _BACKMATTER_TITLE_DENY_LIST:
|
||||
pass # let existing paragraph_title or fallthrough handle it
|
||||
elif raw_label == "doc_title" or (
|
||||
raw_label != "text"
|
||||
and len(block_bbox_local) >= 4 and block_bbox_local[1] < page_height * 0.25
|
||||
and len(_tv) > 20
|
||||
and "abstract" not in _lv[:15]
|
||||
and "keywords" not in _lv[:15]
|
||||
and "introduction" not in _lv[:15]
|
||||
):
|
||||
if len(_tv) > 20 and not _seems_like_authors(_tv):
|
||||
return RoleAssignment(
|
||||
role="paper_title",
|
||||
confidence=0.6,
|
||||
evidence=[f"page-1 frontmatter title guard: {_tv[:60]}"],
|
||||
)
|
||||
|
||||
# Paddle priors
|
||||
if raw_label == "paragraph_title":
|
||||
stripped = text.strip().lstrip("*•·-–—")
|
||||
|
|
@ -499,7 +530,7 @@ def assign_block_role(
|
|||
)
|
||||
# Author-like heading guard: prevent bylines from becoming headings
|
||||
_is_author_byline = (
|
||||
re.search(r"&|,.*,", text)
|
||||
(re.search(r"&|,.*,", text) or _seems_like_authors(text))
|
||||
and len(text.split()) <= 15
|
||||
and not _has_heading_numbering(text)
|
||||
and not any(text.lower().startswith(w) for w in ["abstract", "introduction", "methods", "results", "discussion", "conclusion", "references"])
|
||||
|
|
|
|||
|
|
@ -141,3 +141,78 @@ def test_resolve_metadata_author_no_zotero_fallback() -> None:
|
|||
resolved = resolve_metadata(source, candidates)
|
||||
assert resolved["authors"]["source"] == "ocr_frontmatter"
|
||||
assert resolved["authors"]["value"] == ["Alice Smith", "Bob Jones"]
|
||||
|
||||
|
||||
def test_title_anchored_from_source_metadata() -> None:
|
||||
from paperforge.worker.ocr_metadata import (
|
||||
_align_frontmatter_to_source_metadata,
|
||||
)
|
||||
|
||||
source_meta = {
|
||||
"title": "Correct Research Title That Is Long Enough",
|
||||
"authors": ["Alice Smith", "Bob Jones"],
|
||||
}
|
||||
page1_blocks = [
|
||||
{
|
||||
"block_id": "p1_b1",
|
||||
"block_label": "doc_title",
|
||||
"block_content": "Correct Research Title That Is Long Enough",
|
||||
"block_bbox": [100, 50, 900, 100],
|
||||
"page": 1,
|
||||
}
|
||||
]
|
||||
|
||||
aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks)
|
||||
assert aligned["title"]["source"] == "zotero"
|
||||
assert aligned["title"]["value"] == "Correct Research Title That Is Long Enough"
|
||||
assert aligned["title"].get("ocr_aligned") is True
|
||||
|
||||
|
||||
def test_author_anchored_from_source_metadata() -> None:
|
||||
from paperforge.worker.ocr_metadata import (
|
||||
_align_frontmatter_to_source_metadata,
|
||||
)
|
||||
|
||||
source_meta = {
|
||||
"title": "Some Title",
|
||||
"authors": ["Alice Smith", "Bob Jones"],
|
||||
}
|
||||
page1_blocks = [
|
||||
{
|
||||
"block_id": "p1_b2",
|
||||
"block_label": "text",
|
||||
"block_content": "Alice Smith, Bob Jones",
|
||||
"block_bbox": [100, 200, 800, 230],
|
||||
"page": 1,
|
||||
}
|
||||
]
|
||||
|
||||
aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks)
|
||||
assert aligned["authors"]["source"] == "zotero"
|
||||
assert aligned["authors"]["value"] == ["Alice Smith", "Bob Jones"]
|
||||
assert aligned["authors"].get("ocr_aligned") is True
|
||||
|
||||
|
||||
def test_mismatched_ocr_title_does_not_pollute_metadata() -> None:
|
||||
from paperforge.worker.ocr_metadata import (
|
||||
_align_frontmatter_to_source_metadata,
|
||||
)
|
||||
|
||||
source_meta = {
|
||||
"title": "Correct Zotero Title That Is Real",
|
||||
"authors": ["Alice Smith"],
|
||||
}
|
||||
page1_blocks = [
|
||||
{
|
||||
"block_id": "p1_b1",
|
||||
"block_label": "doc_title",
|
||||
"block_content": "Totally Wrong OCR Title That Should Not Be Used",
|
||||
"block_bbox": [100, 50, 900, 100],
|
||||
"page": 1,
|
||||
}
|
||||
]
|
||||
|
||||
aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks)
|
||||
assert aligned["title"]["value"] == "Correct Zotero Title That Is Real"
|
||||
assert aligned["title"]["source"] == "zotero"
|
||||
assert aligned["title"].get("ocr_aligned", False) is False
|
||||
|
|
|
|||
|
|
@ -533,6 +533,43 @@ def test_running_header_not_heading():
|
|||
assert result.role == "noise", f"Expected noise, got {result.role}"
|
||||
|
||||
|
||||
def test_doc_title_not_body_paragraph() -> None:
|
||||
from paperforge.worker.ocr_roles import assign_block_role
|
||||
|
||||
block = {
|
||||
"block_label": "doc_title",
|
||||
"block_content": "A Very Long Title That Would Normally Be Considered A Document Title Block With Sufficient Length",
|
||||
"block_bbox": [100, 50, 900, 100],
|
||||
"page": 1,
|
||||
}
|
||||
result = assign_block_role(block, page_blocks=[block], page_width=900, page_height=1200)
|
||||
assert result.role != "body_paragraph", (
|
||||
f"doc_title should not be body_paragraph, got {result.role}"
|
||||
)
|
||||
|
||||
|
||||
def test_author_byline_comma_only_not_section_heading() -> None:
|
||||
"""Comma-separated byline on page 1 below zone detection threshold -> NOT heading."""
|
||||
from paperforge.worker.ocr_roles import assign_block_role
|
||||
|
||||
block = {
|
||||
"block_label": "paragraph_title",
|
||||
"block_content": "John Smith, Jane Doe",
|
||||
"block_bbox": [100, 400, 500, 430],
|
||||
"page": 1,
|
||||
}
|
||||
page_blocks = [block]
|
||||
result = assign_block_role(
|
||||
block,
|
||||
page_blocks=page_blocks,
|
||||
page_width=600,
|
||||
page_height=800,
|
||||
)
|
||||
assert result.role not in ("section_heading", "subsection_heading", "sub_subsection_heading"), (
|
||||
f"Comma-separated byline should not be a heading, got {result.role}"
|
||||
)
|
||||
|
||||
|
||||
def test_backmatter_boundary_detects_on_early_page() -> None:
|
||||
"""Backmatter boundary should be detectable on papers with fewer
|
||||
than 8 pages, without a hard page gate."""
|
||||
|
|
|
|||
Loading…
Reference in a new issue