From 8a0f6dd0fb7b28e57a83280167eef3cefcb49b13 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Fri, 5 Jun 2026 23:27:10 +0800 Subject: [PATCH] =?UTF-8?q?feat(ocr):=20PeerJ-style=20remediation=20?= =?UTF-8?q?=E2=80=94=20frontmatter=20zoning,=20heading=20hierarchy,=20back?= =?UTF-8?q?matter=20container?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Task 1: Strengthen first-page frontmatter zoning (author/affiliation/furniture zones) - Task 2: Style-aware heading hierarchy with _infer_heading_level content heuristics - Tasks 3+4: ADDITIONAL INFORMATION AND DECLARATIONS as backmatter_boundary_heading, container ordering (boundary heading emitted first, sub-sections after) - Task 5: Figure legend validation via _is_formal_legend (width/criterion/axis-vocab rejection) - Verified on 2GN9LMCW: authors correct, furniture suppressed, heading hierarchy, references last - 7C8829BD regression-free: 26/26 markers, tail order intact - 296/296 tests pass --- paperforge/worker/ocr_figures.py | 71 ++++++- paperforge/worker/ocr_metadata.py | 52 ++++-- paperforge/worker/ocr_render.py | 147 +++++++++++---- paperforge/worker/ocr_roles.py | 298 +++++++++++++++++++++++++++++- 4 files changed, 510 insertions(+), 58 deletions(-) diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 879de61f..dc562920 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -85,18 +85,86 @@ def _is_body_mention(block: dict) -> bool: return False -def build_figure_inventory(structured_blocks: list[dict]) -> dict[str, Any]: +def _is_formal_legend(text: str, block: dict | None = None, page_width: float = 1200) -> bool: + if not text: + return False + + if _FIGURE_NUMBER_PATTERN.search(text): + return True + + if block is not None: + bbox = block.get("bbox") or block.get("block_bbox") + if bbox and len(bbox) >= 4: + block_width = bbox[2] - bbox[0] + if block_width < page_width * 0.3: + return False + + lower = text.lower().strip() + axis_words = { + "days", + "time", + "concentration", + "percentage", + "volume", + "frequency", + "intensity", + "ratio", + "expression", + "level", + "content", + "activity", + "treatment", + "group", + "control", + "dose", + "response", + "size", + } + words = set(lower.rstrip(". ").split()) + stop_words = { + "of", + "the", + "in", + "and", + "to", + "a", + "an", + "by", + "at", + "for", + "with", + "on", + "is", + "are", + "was", + "were", + } + if len(text) < 60 and words and words.issubset(axis_words | stop_words): + return False + + return True + + +def build_figure_inventory(structured_blocks: list[dict], page_width: float = 1200) -> dict[str, Any]: legends: list[dict] = [] + rejected_legends: list[dict] = [] assets: list[dict] = [] unmatched_legends: list[dict] = [] unmatched_assets: list[dict] = [] matched_figures: list[dict] = [] + for block in structured_blocks: + if block.get("page_width"): + page_width = float(block["page_width"]) + for block in structured_blocks: role = block.get("role", "") if role == "figure_caption": if _is_body_mention(block): continue + if not _is_formal_legend(block.get("text", ""), block, page_width): + rejected_legends.append(block) + continue legends.append(block) elif role == "figure_asset": assets.append(block) @@ -187,6 +255,7 @@ def build_figure_inventory(structured_blocks: list[dict]) -> dict[str, Any]: "matched_figures": matched_figures, "unmatched_legends": unmatched_legends, "unmatched_assets": unmatched_assets, + "rejected_legends": rejected_legends, "official_figure_count": len(matched_figures), } diff --git a/paperforge/worker/ocr_metadata.py b/paperforge/worker/ocr_metadata.py index fce835e5..782c55a4 100644 --- a/paperforge/worker/ocr_metadata.py +++ b/paperforge/worker/ocr_metadata.py @@ -10,17 +10,35 @@ from paperforge.core.io import write_json def _clean_author_display(author_text: str) -> str: text = author_text - text = re.sub(r'\$\s+\^', '$^', text) - text = re.sub(r'\^\s+\{', '^{', text) - text = re.sub(r'\}\s+\$', '}$', text) - text = re.sub(r',(\s*)and(\s*)', r', and ', text) - text = re.sub(r';(\s*)and(\s*)', r'; and ', text) - text = re.sub(r'([,;:])\1+', r'\1', text) - text = re.sub(r'\s{2,}', ' ', text) - text = text.strip().strip(';, ') + text = re.sub(r"\$\s+\^", "$^", text) + text = re.sub(r"\^\s+\{", "^{", text) + text = re.sub(r"\}\s+\$", "}$", text) + text = re.sub(r",(\s*)and(\s*)", r", and ", text) + text = re.sub(r";(\s*)and(\s*)", r"; and ", text) + text = re.sub(r"([,;:])\1+", r"\1", text) + text = re.sub(r"\s{2,}", " ", text) + text = text.strip().strip(";, ") return text.strip() +def _name_likeness_score(text: str) -> int: + """Score how much the text looks like a human-name list vs affiliation/noise.""" + if not text: + return 0 + score = 0 + if text.count(",") >= 1: + score += text.count(",") + if bool(re.search(r"[A-Z][a-z]+,\s+[A-Z]", text)): + score += 3 + if bool(re.search(r"\band\b\s+[A-Z]", text)): + score += 2 + if bool(re.search(r"[\*†‡§¶#]", text)): + score += 1 + if any(kw in text.lower() for kw in ["university", "department", "institute", "college", "school of"]): + score -= 5 + return score + + def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, Any]: candidates: dict[str, Any] = { "title": None, @@ -47,7 +65,13 @@ def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, An if role == "paper_title": candidates["title"] = text elif role == "authors": - candidates["authors_text"] = text + if candidates["authors_text"] is None: + candidates["authors_text"] = text + else: + current_score = _name_likeness_score(candidates["authors_text"]) + new_score = _name_likeness_score(text) + if new_score > current_score: + candidates["authors_text"] = text elif role in ("affiliation",): if "affiliation_blocks" not in candidates: candidates["affiliation_blocks"] = [] @@ -114,6 +138,12 @@ def resolve_metadata( # --- authors --- zotero_authors = source_metadata.get("authors", []) ocr_authors_text = frontmatter_candidates.get("authors_text", "") + + _INST_KEYWORDS = ["university", "department", "institute", "college", "school of"] + + def _is_affiliation_like(t: str) -> bool: + return any(kw in t.lower() for kw in _INST_KEYWORDS) + ocr_author_list = ( [a.strip() for a in re.split(r",\s+(?=[A-Z])", ocr_authors_text) if a.strip()] if ocr_authors_text else [] ) @@ -124,7 +154,7 @@ def resolve_metadata( "source": "zotero", "confidence": 0.99, } - elif ocr_author_list: + elif ocr_author_list and not _is_affiliation_like(ocr_authors_text): resolved["authors"] = { "value": ocr_author_list, "source": "ocr_frontmatter", @@ -140,7 +170,7 @@ def resolve_metadata( # --- authors_display (cleaned string for UI) --- if isinstance(zotero_authors, list) and len(zotero_authors) > 0: resolved["authors_display"] = ", ".join(zotero_authors) - elif ocr_authors_text: + elif ocr_authors_text and not _is_affiliation_like(ocr_authors_text): resolved["authors_display"] = _clean_author_display(ocr_authors_text) else: resolved["authors_display"] = "" diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index 051f6c72..c9119263 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -25,6 +25,7 @@ def _is_bogus_heading(text: str) -> bool: _TAIL_ROLES = frozenset( { + "backmatter_boundary_heading", "backmatter_heading", "backmatter_body", "tail_candidate_body", @@ -206,19 +207,37 @@ def _extract_style_profile(block: dict) -> dict | None: def _build_heading_style_profiles(blocks: list[dict]) -> dict: - heading_roles = frozenset({"section_heading", "subsection_heading", "backmatter_heading", "reference_heading"}) - profiles_with_sizes = [] + heading_roles = frozenset( + { + "backmatter_boundary_heading", + "section_heading", + "subsection_heading", + "sub_subsection_heading", + "backmatter_heading", + "reference_heading", + } + ) + heading_items = [] for block in blocks: if block.get("role") in heading_roles: profile = _extract_style_profile(block) if profile is not None: - profiles_with_sizes.append(profile["mean_size"]) + heading_items.append( + { + "max_size": profile["max_size"], + "mean_size": profile["mean_size"], + "profile": profile, + "block": block, + } + ) - if len(profiles_with_sizes) < 3: + if len(heading_items) < 3: return {} - unique_sizes = sorted(set(profiles_with_sizes)) + # Use max_size for clustering — section headings often have slightly larger + # first characters, making max_size more discriminating than mean. + unique_sizes = sorted(set(item["max_size"] for item in heading_items)) if not unique_sizes: return {} @@ -232,33 +251,70 @@ def _build_heading_style_profiles(blocks: list[dict]) -> dict: current = [s] clusters.append(current) - # Build full profiles for each cluster - cluster_profiles = [] - for cluster in clusters: - matching = [] - for block in blocks: - if block.get("role") in heading_roles: - p = _extract_style_profile(block) - if p is not None and any(abs(p["mean_size"] - s) <= 2 for s in cluster): - matching.append(p) - cluster_profiles.append(matching) + # Build full profile data for each cluster + cluster_items = [] + for cluster_sizes in clusters: + matching = [item for item in heading_items if any(abs(item["max_size"] - s) <= 2 for s in cluster_sizes)] + cluster_items.append(matching) - clusters.sort(key=lambda c: sum(c) / len(c), reverse=True) + # Sort clusters by max_size descending to establish hierarchy level + clusters.sort(key=lambda c: max(c), reverse=True) + cluster_items.sort(key=lambda items: max(i["max_size"] for i in items), reverse=True) - keys = ["primary", "subsection", "backmatter", "body"] + keys = ["primary", "subsection", "sub_subsection", "backmatter", "body"] result = {} - for i, (cluster, profiles) in enumerate(zip(clusters, cluster_profiles, strict=False)): + for i, (cluster_sizes, items) in enumerate(zip(clusters, cluster_items, strict=False)): if i >= len(keys): break + + profiles = [item["profile"] for item in items] + is_bold = any(p["is_bold"] for p in profiles) fonts = set() for p in profiles: fonts.update(p["font_families"]) + + # Compute spacing before/after by searching ALL blocks on the same page + space_before_vals = [] + space_after_vals = [] + for item in items: + heading_block = item["block"] + heading_page = heading_block.get("page") + heading_bbox = heading_block.get("bbox") or heading_block.get("block_bbox") + if heading_bbox and len(heading_bbox) >= 4 and heading_page is not None: + h_y1 = heading_bbox[1] + h_y2 = heading_bbox[3] + + nearest_above_bottom = None + nearest_below_top = None + for other in blocks: + if other is heading_block: + continue + if other.get("page") == heading_page: + obbox = other.get("bbox") or other.get("block_bbox") + if obbox and len(obbox) >= 4: + o_y2 = obbox[3] + o_y1 = obbox[1] + if o_y2 <= h_y1 and (nearest_above_bottom is None or o_y2 > nearest_above_bottom): + nearest_above_bottom = o_y2 + if o_y1 >= h_y2 and (nearest_below_top is None or o_y1 < nearest_below_top): + nearest_below_top = o_y1 + + if nearest_above_bottom is not None: + space_before_vals.append(h_y1 - nearest_above_bottom) + if nearest_below_top is not None: + space_after_vals.append(nearest_below_top - h_y2) + + space_before = sum(space_before_vals) / len(space_before_vals) if space_before_vals else 0.0 + space_after = sum(space_after_vals) / len(space_after_vals) if space_after_vals else 0.0 + result[keys[i]] = { - "size_min": min(cluster), - "size_max": max(cluster), + "size_min": min(cluster_sizes), + "size_max": max(cluster_sizes), "bold": is_bold, "fonts": fonts, + "space_before": space_before, + "space_after": space_after, } return result @@ -271,21 +327,24 @@ def _disambiguate_heading_role(block: dict, style_profiles: dict) -> str | None: size = profile["mean_size"] - for role_key in ("primary", "subsection", "backmatter", "body"): + for role_key, role_name in [ + ("primary", "section_heading"), + ("subsection", "subsection_heading"), + ("sub_subsection", "sub_subsection_heading"), + ("backmatter", "backmatter_heading"), + ("body", None), + ]: cfg = style_profiles.get(role_key) if cfg is None: continue if cfg["size_min"] <= size <= cfg["size_max"]: - if role_key == "primary": - return "section_heading" - elif role_key == "subsection": - return "subsection_heading" - elif role_key == "backmatter": + if role_key == "backmatter": if profile["is_bold"]: - return "backmatter_heading" + return role_name return None elif role_key == "body": return None + return role_name return None @@ -335,7 +394,7 @@ def _reorder_tail_run( body_pool: list[dict] = [] for block in tail_blocks: role = block.get("role") - if role == "backmatter_heading": + if role in ("backmatter_boundary_heading", "backmatter_heading"): backmatter_sections.append({"heading": block, "bodies": []}) elif role == "reference_heading": ref_section = {"heading": block, "bodies": []} @@ -407,11 +466,16 @@ def _reorder_tail_run( else: orphan_blocks.append(body) - # Phase 5 — emit: non-tail pass (body/unknown), then backmatter, then references + # Phase 5 — emit: non-tail pass, then boundary heading first + # (backmatter container), then sub-sections, then references result: list[dict] = [] result.extend(non_tail_pass) result.extend(carried_bodies) - for sec in backmatter_sections: + boundary_secs = [s for s in backmatter_sections + if s["heading"].get("role") == "backmatter_boundary_heading"] + sub_secs = [s for s in backmatter_sections + if s["heading"].get("role") != "backmatter_boundary_heading"] + for sec in boundary_secs + sub_secs: result.append(sec["heading"]) result.extend(sec["bodies"]) if ref_section is not None and ref_section is not carried_ref: @@ -442,7 +506,7 @@ def _reorder_tail_run_fifo( for block in tail_blocks: role = block.get("role") - if role == "backmatter_heading": + if role in ("backmatter_boundary_heading", "backmatter_heading"): sec = {"heading": block, "bodies": []} backmatter_sections.append(sec) heading_queue.append(sec) @@ -471,7 +535,11 @@ def _reorder_tail_run_fifo( result: list[dict] = [] result.extend(non_tail_pass) - for sec in backmatter_sections: + boundary_secs = [s for s in backmatter_sections + if s["heading"].get("role") == "backmatter_boundary_heading"] + sub_secs = [s for s in backmatter_sections + if s["heading"].get("role") != "backmatter_boundary_heading"] + for sec in boundary_secs + sub_secs: result.append(sec["heading"]) result.extend(sec["bodies"]) if ref_section is not None and ref_section is not carried_ref: @@ -511,7 +579,9 @@ def _promote_tail_body_candidates( continue page_blocks = [result[i] for i in indices] - local_headings = [b for b in page_blocks if b.get("role") == "backmatter_heading"] + local_headings = [ + b for b in page_blocks if b.get("role") in ("backmatter_heading", "backmatter_boundary_heading") + ] ref_heading = next((b for b in page_blocks if b.get("role") == "reference_heading"), None) local_anchors = local_headings local_tops = [] @@ -634,7 +704,7 @@ def _detect_forward_body_end(blocks: list[dict]) -> int | None: for page in pages: roles = {b.get("role") for b in by_page[page]} - has_body = bool(roles & {"body_paragraph", "section_heading", "subsection_heading"}) + has_body = bool(roles & {"body_paragraph", "section_heading", "subsection_heading", "sub_subsection_heading"}) has_tail = bool(roles & _TAIL_ROLES) if has_body and not has_tail: @@ -674,7 +744,7 @@ def _detect_backward_backmatter_start(blocks: list[dict]) -> int | None: page_blocks = by_page[page] roles = {b.get("role") for b in page_blocks} - if "reference_heading" in roles or "backmatter_heading" in roles: + if "reference_heading" in roles or "backmatter_heading" in roles or "backmatter_boundary_heading" in roles: return page dense_refs = sum(1 for b in page_blocks if b.get("role") == "reference_item") @@ -731,7 +801,8 @@ def _assign_tail_spread_ownership( with _spread_anchor so the page-local reorder pass does not reassign them. Unanchored candidates inside the spread stay as tail_candidate_body. """ - anchors = [b for b in blocks if b.get("role") == "backmatter_heading"] + tail_heading_roles = {"backmatter_heading", "backmatter_boundary_heading"} + anchors = [b for b in blocks if b.get("role") in tail_heading_roles] ref_heading = next((b for b in blocks if b.get("role") == "reference_heading"), None) if tail_spread is not None: @@ -1007,7 +1078,7 @@ def render_fulltext_markdown( lines.append("") emitted_pages.add(block_page) - if role == "backmatter_heading" or role == "reference_heading": + if role == "backmatter_boundary_heading" or role == "backmatter_heading" or role == "reference_heading": lines.append(f"## {text}") lines.append("") elif role == "section_heading": @@ -1020,7 +1091,7 @@ def render_fulltext_markdown( else: lines.append(f"## {text}") lines.append("") - elif role == "subsection_heading": + elif role == "subsection_heading" or role == "sub_subsection_heading": lines.append(f"### {text}") lines.append("") elif role in ("backmatter_body", "tail_candidate_body", "body_paragraph"): diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index 0d29f6ba..cb56134a 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -109,6 +109,226 @@ def _looks_like_reference(text: str) -> bool: return bool(_REFERENCE_PATTERN.match(text.strip())) +def _is_backmatter_boundary_heading(block: dict, page_blocks: list[dict]) -> bool: + text = str(block.get("block_content", "") or "").strip() + if not text: + return False + + page_num = block.get("page", 1) or 1 + if page_num < 8: + return False + + raw_label = str(block.get("block_label", "") or "").strip() + + is_heading_label = raw_label == "paragraph_title" + span_meta = block.get("span_metadata", {}) or {} + is_visually_heading = False + if isinstance(span_meta, dict): + font_size = span_meta.get("size", 0) or 0 + font_flags = (span_meta.get("flags", "") or "").lower() + is_visually_heading = ("bold" in font_flags and font_size >= 11) or font_size >= 14 + elif isinstance(span_meta, list): + sizes = [s.get("size", 0) or 0 for s in span_meta if isinstance(s, dict)] + bold_flags = any(s.get("flags", 0) & 16 for s in span_meta if isinstance(s, dict)) + avg_size = sum(sizes) / len(sizes) if sizes else 0 + is_visually_heading = bold_flags or avg_size >= 14 + + if not (is_heading_label or is_visually_heading): + return False + + upper = text.upper() + has_container_words = ( + "ADDITIONAL" in upper or "SUPPLEMENTARY" in upper or "DECLARATION" in upper or "INFORMATION" in upper + ) + if not has_container_words: + return False + + if len(text) <= 20: + return False + + lower = text.lower() + if lower in ("references", "bibliography"): + return False + if lower in _BACKMATTER_HEADINGS: + return False + return lower not in _BACKMATTER_TITLE_DENY_LIST + + +def _looks_like_author_list(text: str) -> bool: + """Check if text looks like a list of author names (not a title or body).""" + if not text: + return False + has_name_comma = bool(re.search(r"[A-Z][a-z]+,\s+[A-Z]", text)) + has_and_name = bool(re.search(r"\band\b\s+[A-Z][a-z]+", text)) + has_author_marker = bool(re.search(r"[\*†‡§¶#]", text)) + has_two_name_pairs = bool(re.search(r"[A-Z][a-z]+\s+[A-Z][a-z]+\s*[·•,;]\s*[A-Z][a-z]+", text)) + has_et_al = "et al" in text.lower() + return (has_name_comma or has_and_name or has_author_marker or has_two_name_pairs or has_et_al) and len(text) < 500 + + +def _looks_like_affiliation(text: str) -> bool: + """Check if text looks like an affiliation block.""" + lower_txt = text.lower() + inst_keywords = [ + "university", + "department", + "institute", + "college", + "school of", + "faculty", + "laboratory", + "lab", + "hospital", + "center", + "centre", + "academy", + "division", + "initiative", + "regenerative medicine", + "research", + "science", + "technology", + "medicine", + "school of materials", + ] + has_inst = any(kw in lower_txt for kw in inst_keywords) + has_city_country = bool( + re.search( + r"(?:,\s*)(?:USA|UK|China|Germany|France|Japan|Italy|Canada|" + r"Australia|Brazil|India|Korea|Spain|Netherlands|Switzerland|" + r"Sweden|Norway|Denmark|Austria|Belgium|Finland|Poland|" + r"Russia|Mexico|Argentina|Singapore|Taiwan|Hong\s*Kong)", + text, + ) + ) + has_number_prefix = bool(re.match(r"^[\$\s]*\^?\d+\s*", text)) + has_superscript_number = bool(re.search(r"\$?\^\d+\$?", text)) + return has_inst or (has_city_country and has_number_prefix) or has_superscript_number + + +def _infer_heading_level(text: str, font_size: float = 0) -> str: + """Infer heading level for unnumbered papers using text content heuristics. + + Uses case patterns, word count, and optional font size to distinguish + top-level section headings from lower-level sub-sections. Font size + overrides content heuristics when >= 14pt (clearly a major heading). + """ + if not text: + return "section_heading" + if font_size >= 14: + return "section_heading" + upper_ratio = sum(1 for c in text if c.isupper()) / max(len(text), 1) + is_mostly_upper = upper_ratio > 0.7 + sentence_verbs = {" is ", " are ", " was ", " were ", " have ", " has ", " been "} + has_sentence_verb = any(v in text.lower() for v in sentence_verbs) + word_count = len(text.split()) + if is_mostly_upper and word_count >= 1 and not has_sentence_verb: + return "section_heading" + if word_count >= 2 and not has_sentence_verb: + return "subsection_heading" + if word_count == 1 and not has_sentence_verb: + return "sub_subsection_heading" + return "section_heading" + + +def _detect_frontmatter_zone( + block: dict, + page_blocks: list[dict], + page_height: float, + page_width: float, + style_profiles: dict | None = None, +) -> str | None: + """Detect frontmatter zone for a block on page 1. + + Returns one of: ``title_zone``, ``author_zone``, ``affiliation_zone``, + ``journal_furniture_zone``, ``abstract_zone``, or ``None``. + """ + page_num = block.get("page", 1) or 1 + if page_num > 1: + return None + + bbox = block.get("block_bbox", [0, 0, 0, 0]) + if len(bbox) < 4: + return None + + text = str(block.get("block_content", "") or "").strip() + if not text: + return None + + lower_txt = text.lower() + raw_label = str(block.get("block_label", "") or "").strip() + x1, y1, x2 = bbox[0], bbox[1], bbox[2] + + # --- abstract_zone --- + if lower_txt.startswith("abstract") and len(text) < 30: + return "abstract_zone" + + # --- journal_furniture_zone --- + furniture_signals = [ + "submitted", + "accepted", + "published", + "received", + "copyright", + "©", + "doi:", + "doi ", + "https://doi.org", + "academic editor", + "how to cite", + "to cite this article", + "creative commons", + "cc by", + "cc license", + "this is an open-access article", + "reviewed by", + "edited by", + "present address", + ] + if any(s in lower_txt for s in furniture_signals): + return "journal_furniture_zone" + + narrow_furniture = [ + "citation:", + "correspondence", + "orcid", + "these authors contributed equally", + "equal contribution", + "additional information", + ] + if any(s in lower_txt for s in narrow_furniture): + block_width = x2 - x1 + is_narrow = page_width > 0 and block_width < page_width * 0.35 + is_top_half = page_height > 0 and y1 < page_height * 0.5 + if is_narrow or is_top_half: + return "journal_furniture_zone" + + # --- title_zone --- + if page_height > 0 and y1 < page_height * 0.2: + block_width = x2 - x1 + is_wide_enough = page_width <= 0 or block_width > page_width * 0.4 + if is_wide_enough and lower_txt not in _BACKMATTER_TITLE_DENY_LIST and not _looks_like_author_list(text): + if raw_label in ("paragraph_title", "doc_title"): + return "title_zone" + if raw_label == "text" and len(text) < 80: + return "title_zone" + + # --- author_zone --- + if ( + page_height > 0 + and y1 < page_height * 0.4 + and _looks_like_author_list(text) + and not _looks_like_affiliation(text) + ): + return "author_zone" + + # --- affiliation_zone --- + if page_height > 0 and y1 < page_height * 0.6 and _looks_like_affiliation(text): + return "affiliation_zone" + + return None + + def assign_block_role( block: dict, page_blocks: list[dict], @@ -147,6 +367,50 @@ def assign_block_role( evidence=[f"table prefix matched: {text[:60]}"], ) + # --- Page-1 frontmatter zone pre-filter --- + page_num = block.get("page", 1) or 1 + zone = None + if page_num == 1 and raw_label in ("paragraph_title", "text"): + zone = _detect_frontmatter_zone( + block, + page_blocks, + page_height, + page_width, + style_profiles, + ) + + if zone == "title_zone": + lower_stripped = text.strip().lstrip("*•·-–—").lower() + if lower_stripped not in _BACKMATTER_TITLE_DENY_LIST: + return RoleAssignment( + role="paper_title", + confidence=0.8, + evidence=[f"page-1 zone title_zone: {text[:60]}"], + ) + + if zone == "author_zone": + return RoleAssignment( + role="authors", + confidence=0.8, + evidence=[f"page-1 zone author_zone: {text[:60]}"], + ) + + if zone == "affiliation_zone": + return RoleAssignment( + role="affiliation", + confidence=0.8, + evidence=[f"page-1 zone affiliation_zone: {text[:60]}"], + ) + + if zone == "journal_furniture_zone": + return RoleAssignment( + role="frontmatter_noise", + confidence=0.8, + evidence=[f"page-1 zone journal_furniture_zone: {text[:60]}"], + ) + + # zone == "abstract_zone" → let existing logic handle + # Paddle priors if raw_label == "paragraph_title": stripped = text.strip().lstrip("*•·-–—") @@ -187,6 +451,14 @@ def assign_block_role( confidence=0.5, evidence=[f"backmatter heading text on page 1, treated as section: {text[:60]}"], ) + # Backmatter boundary container heading (e.g. "ADDITIONAL INFORMATION AND DECLARATIONS") + # Checked after specific heading detection but before numbered/generic fallback. + if _is_backmatter_boundary_heading(block, page_blocks): + return RoleAssignment( + role="backmatter_boundary_heading", + confidence=0.7, + evidence=[f"backmatter boundary heading: {text[:60]}"], + ) if _has_heading_numbering(text): return RoleAssignment( role="section_heading" if re.match(r"^\d+\s", text) else "subsection_heading", @@ -199,7 +471,7 @@ def assign_block_role( if bbox[1] < max(page_height, 1) * 0.25: if lower in _BACKMATTER_TITLE_DENY_LIST: return RoleAssignment( - role="section_heading", + role="section_heading" if re.match(r"^\d+\s", text) else "subsection_heading", confidence=0.5, evidence=[f"backmatter title in title zone, treated as heading: {text[:60]}"], ) @@ -213,10 +485,11 @@ def assign_block_role( confidence=0.5, evidence=[f"unnumbered paragraph_title on page 1 outside title zone: {text[:60]}"], ) + level = _infer_heading_level(text) return RoleAssignment( - role="section_heading", + role=level, confidence=0.6, - evidence=[f"unnumbered paragraph_title on page {page_num}, treated as heading: {text[:60]}"], + evidence=[f"unnumbered paragraph_title, inferred level {level}: {text[:60]}"], ) if raw_label == "figure_title": @@ -401,7 +674,7 @@ def assign_block_role( no_sentence_verbs = not (len(text) > 50 and any(v in text.lower() for v in sentence_verbs)) if in_top_80 and wide_enough and no_sentence_verbs: return RoleAssignment( - role="section_heading" if re.match(r"^\d+\s", text) else "subsection_heading", + role="section_heading", confidence=0.65, evidence=[f"numbered text block: {text[:60]}"], ) @@ -413,7 +686,7 @@ def assign_block_role( evidence=[f"references heading from text block: {text[:40]}"], ) - # Style-aware heading disambiguation (Task 5) + # Style-aware heading disambiguation — primary signal for unnumbered papers if style_profiles is not None and style_profiles: from paperforge.worker.ocr_render import _disambiguate_heading_role @@ -424,7 +697,7 @@ def assign_block_role( if in_top_80 and len(text) >= 5 and len(text) < 60 and ". " not in text: return RoleAssignment( role=style_suggested, - confidence=0.65, + confidence=0.7, evidence=[f"style-aware heading detection: role={style_suggested}, text={text[:40]}"], ) @@ -441,12 +714,21 @@ def assign_block_role( bbox = block.get("block_bbox", [0, 0, 0, 0]) in_top_80 = page_height == 0 or (len(bbox) >= 4 and bbox[1] < page_height * 0.8) if in_top_80: + heading_level = _infer_heading_level(text, font_size) return RoleAssignment( - role="section_heading", + role=heading_level, confidence=0.65, - evidence=[f"heading-style text block: size={font_size}, flags={font_flags}, text={text[:40]}"], + evidence=[f"heading-style text block: size={font_size}, flags={font_flags}, level={heading_level}, text={text[:40]}"], ) + # Backmatter boundary container heading detection for text blocks + if _is_backmatter_boundary_heading(block, page_blocks): + return RoleAssignment( + role="backmatter_boundary_heading", + confidence=0.7, + evidence=[f"backmatter boundary heading from text block: {text[:60]}"], + ) + if len(text) < 20: return RoleAssignment( role="unknown_structural",