diff --git a/paperforge/worker/ocr_metadata.py b/paperforge/worker/ocr_metadata.py index 9319534b..c299216e 100644 --- a/paperforge/worker/ocr_metadata.py +++ b/paperforge/worker/ocr_metadata.py @@ -129,9 +129,51 @@ def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, An return candidates +def _token_overlap(a: str, b: str) -> float: + ta = set(re.findall(r"[a-zA-Z0-9]+", a)) + tb = set(re.findall(r"[a-zA-Z0-9]+", b)) + if not ta or not tb: + return 0.0 + return len(ta & tb) / min(len(ta), len(tb)) + + +def _align_frontmatter_to_source_metadata( + source_meta: dict, + page1_blocks: list[dict], +) -> dict: + result: dict[str, Any] = {} + + source_title = (source_meta.get("title") or "").strip() + if source_title: + result["title"] = {"source": "zotero", "value": source_title} + for b in page1_blocks: + t = (b.get("text") or b.get("block_content") or "").strip() + if len(t) >= 20 and ( + t in source_title or source_title in t + or _token_overlap(t.lower(), source_title.lower()) > 0.6 + ): + result["title"]["ocr_block_id"] = b.get("block_id") + result["title"]["ocr_aligned"] = True + break + + source_authors = source_meta.get("authors") or [] + if source_authors: + result["authors"] = {"source": "zotero", "value": source_authors} + for b in page1_blocks: + t = (b.get("text") or b.get("block_content") or "").strip() + match = _match_author_block_to_source_authors(t, source_authors) + if match.get("matched"): + result["authors"]["ocr_block_id"] = b.get("block_id") + result["authors"]["ocr_aligned"] = True + break + + return result + + def resolve_metadata( source_metadata: dict[str, Any], frontmatter_candidates: dict[str, Any] | None = None, + page1_blocks: list[dict] | None = None, ) -> dict[str, Any]: if frontmatter_candidates is None: frontmatter_candidates = {} @@ -225,6 +267,16 @@ def resolve_metadata( else: resolved["authors_display"] = "" + # --- frontmatter alignment (page-1 block anchoring) --- + if page1_blocks: + alignment = _align_frontmatter_to_source_metadata(source_metadata, page1_blocks) + if "title" in resolved and "title" in alignment and alignment["title"].get("ocr_aligned"): + resolved["title"]["ocr_block_id"] = alignment["title"]["ocr_block_id"] + resolved["title"]["ocr_aligned"] = True + if "authors" in resolved and "authors" in alignment and alignment["authors"].get("ocr_aligned"): + resolved["authors"]["ocr_block_id"] = alignment["authors"]["ocr_block_id"] + resolved["authors"]["ocr_aligned"] = True + # --- year --- zotero_year = source_metadata.get("year", 0) if zotero_year: diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index fde32c0e..09d3d101 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -306,6 +306,14 @@ def _infer_heading_level( return "section_heading" +def _seems_like_authors(text: str) -> bool: + parts = [p.strip() for p in text.replace(" ", " ").split(",")] + if len(parts) >= 2: + name_count = sum(1 for p in parts if re.search(r"[A-Z][a-z]+\s+[A-Z]", p)) + return name_count >= 2 + return False + + def assign_block_role( block: dict, page_blocks: list[dict], @@ -414,6 +422,29 @@ def assign_block_role( # zone == "abstract_zone" → let existing logic handle + # ---- Page-1 frontmatter guard ---- + _pg = block.get("page", 1) or 1 + if _pg == 1 and raw_label in ("doc_title", "paragraph_title", "text"): + _tv = text + _lv = _tv.lower().strip().lstrip("*•·-–—") + block_bbox_local = block.get("block_bbox", [0, 0, 0, 0]) + if _lv in _BACKMATTER_TITLE_DENY_LIST: + pass # let existing paragraph_title or fallthrough handle it + elif raw_label == "doc_title" or ( + raw_label != "text" + and len(block_bbox_local) >= 4 and block_bbox_local[1] < page_height * 0.25 + and len(_tv) > 20 + and "abstract" not in _lv[:15] + and "keywords" not in _lv[:15] + and "introduction" not in _lv[:15] + ): + if len(_tv) > 20 and not _seems_like_authors(_tv): + return RoleAssignment( + role="paper_title", + confidence=0.6, + evidence=[f"page-1 frontmatter title guard: {_tv[:60]}"], + ) + # Paddle priors if raw_label == "paragraph_title": stripped = text.strip().lstrip("*•·-–—") @@ -499,7 +530,7 @@ def assign_block_role( ) # Author-like heading guard: prevent bylines from becoming headings _is_author_byline = ( - re.search(r"&|,.*,", text) + (re.search(r"&|,.*,", text) or _seems_like_authors(text)) and len(text.split()) <= 15 and not _has_heading_numbering(text) and not any(text.lower().startswith(w) for w in ["abstract", "introduction", "methods", "results", "discussion", "conclusion", "references"]) diff --git a/tests/test_ocr_metadata.py b/tests/test_ocr_metadata.py index 15d69184..438244d7 100644 --- a/tests/test_ocr_metadata.py +++ b/tests/test_ocr_metadata.py @@ -141,3 +141,78 @@ def test_resolve_metadata_author_no_zotero_fallback() -> None: resolved = resolve_metadata(source, candidates) assert resolved["authors"]["source"] == "ocr_frontmatter" assert resolved["authors"]["value"] == ["Alice Smith", "Bob Jones"] + + +def test_title_anchored_from_source_metadata() -> None: + from paperforge.worker.ocr_metadata import ( + _align_frontmatter_to_source_metadata, + ) + + source_meta = { + "title": "Correct Research Title That Is Long Enough", + "authors": ["Alice Smith", "Bob Jones"], + } + page1_blocks = [ + { + "block_id": "p1_b1", + "block_label": "doc_title", + "block_content": "Correct Research Title That Is Long Enough", + "block_bbox": [100, 50, 900, 100], + "page": 1, + } + ] + + aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks) + assert aligned["title"]["source"] == "zotero" + assert aligned["title"]["value"] == "Correct Research Title That Is Long Enough" + assert aligned["title"].get("ocr_aligned") is True + + +def test_author_anchored_from_source_metadata() -> None: + from paperforge.worker.ocr_metadata import ( + _align_frontmatter_to_source_metadata, + ) + + source_meta = { + "title": "Some Title", + "authors": ["Alice Smith", "Bob Jones"], + } + page1_blocks = [ + { + "block_id": "p1_b2", + "block_label": "text", + "block_content": "Alice Smith, Bob Jones", + "block_bbox": [100, 200, 800, 230], + "page": 1, + } + ] + + aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks) + assert aligned["authors"]["source"] == "zotero" + assert aligned["authors"]["value"] == ["Alice Smith", "Bob Jones"] + assert aligned["authors"].get("ocr_aligned") is True + + +def test_mismatched_ocr_title_does_not_pollute_metadata() -> None: + from paperforge.worker.ocr_metadata import ( + _align_frontmatter_to_source_metadata, + ) + + source_meta = { + "title": "Correct Zotero Title That Is Real", + "authors": ["Alice Smith"], + } + page1_blocks = [ + { + "block_id": "p1_b1", + "block_label": "doc_title", + "block_content": "Totally Wrong OCR Title That Should Not Be Used", + "block_bbox": [100, 50, 900, 100], + "page": 1, + } + ] + + aligned = _align_frontmatter_to_source_metadata(source_meta, page1_blocks) + assert aligned["title"]["value"] == "Correct Zotero Title That Is Real" + assert aligned["title"]["source"] == "zotero" + assert aligned["title"].get("ocr_aligned", False) is False diff --git a/tests/test_ocr_roles.py b/tests/test_ocr_roles.py index d8d66bb7..8d74d4cf 100644 --- a/tests/test_ocr_roles.py +++ b/tests/test_ocr_roles.py @@ -533,6 +533,43 @@ def test_running_header_not_heading(): assert result.role == "noise", f"Expected noise, got {result.role}" +def test_doc_title_not_body_paragraph() -> None: + from paperforge.worker.ocr_roles import assign_block_role + + block = { + "block_label": "doc_title", + "block_content": "A Very Long Title That Would Normally Be Considered A Document Title Block With Sufficient Length", + "block_bbox": [100, 50, 900, 100], + "page": 1, + } + result = assign_block_role(block, page_blocks=[block], page_width=900, page_height=1200) + assert result.role != "body_paragraph", ( + f"doc_title should not be body_paragraph, got {result.role}" + ) + + +def test_author_byline_comma_only_not_section_heading() -> None: + """Comma-separated byline on page 1 below zone detection threshold -> NOT heading.""" + from paperforge.worker.ocr_roles import assign_block_role + + block = { + "block_label": "paragraph_title", + "block_content": "John Smith, Jane Doe", + "block_bbox": [100, 400, 500, 430], + "page": 1, + } + page_blocks = [block] + result = assign_block_role( + block, + page_blocks=page_blocks, + page_width=600, + page_height=800, + ) + assert result.role not in ("section_heading", "subsection_heading", "sub_subsection_heading"), ( + f"Comma-separated byline should not be a heading, got {result.role}" + ) + + def test_backmatter_boundary_detects_on_early_page() -> None: """Backmatter boundary should be detectable on papers with fewer than 8 pages, without a hard page gate."""