mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
feat(ocr): PeerJ-style remediation — frontmatter zoning, heading hierarchy, backmatter container
- Task 1: Strengthen first-page frontmatter zoning (author/affiliation/furniture zones) - Task 2: Style-aware heading hierarchy with _infer_heading_level content heuristics - Tasks 3+4: ADDITIONAL INFORMATION AND DECLARATIONS as backmatter_boundary_heading, container ordering (boundary heading emitted first, sub-sections after) - Task 5: Figure legend validation via _is_formal_legend (width/criterion/axis-vocab rejection) - Verified on 2GN9LMCW: authors correct, furniture suppressed, heading hierarchy, references last - 7C8829BD regression-free: 26/26 markers, tail order intact - 296/296 tests pass
This commit is contained in:
parent
d52ea37e92
commit
8a0f6dd0fb
4 changed files with 510 additions and 58 deletions
|
|
@ -85,18 +85,86 @@ def _is_body_mention(block: dict) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def build_figure_inventory(structured_blocks: list[dict]) -> dict[str, Any]:
|
||||
def _is_formal_legend(text: str, block: dict | None = None, page_width: float = 1200) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
|
||||
if _FIGURE_NUMBER_PATTERN.search(text):
|
||||
return True
|
||||
|
||||
if block is not None:
|
||||
bbox = block.get("bbox") or block.get("block_bbox")
|
||||
if bbox and len(bbox) >= 4:
|
||||
block_width = bbox[2] - bbox[0]
|
||||
if block_width < page_width * 0.3:
|
||||
return False
|
||||
|
||||
lower = text.lower().strip()
|
||||
axis_words = {
|
||||
"days",
|
||||
"time",
|
||||
"concentration",
|
||||
"percentage",
|
||||
"volume",
|
||||
"frequency",
|
||||
"intensity",
|
||||
"ratio",
|
||||
"expression",
|
||||
"level",
|
||||
"content",
|
||||
"activity",
|
||||
"treatment",
|
||||
"group",
|
||||
"control",
|
||||
"dose",
|
||||
"response",
|
||||
"size",
|
||||
}
|
||||
words = set(lower.rstrip(". ").split())
|
||||
stop_words = {
|
||||
"of",
|
||||
"the",
|
||||
"in",
|
||||
"and",
|
||||
"to",
|
||||
"a",
|
||||
"an",
|
||||
"by",
|
||||
"at",
|
||||
"for",
|
||||
"with",
|
||||
"on",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
}
|
||||
if len(text) < 60 and words and words.issubset(axis_words | stop_words):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def build_figure_inventory(structured_blocks: list[dict], page_width: float = 1200) -> dict[str, Any]:
|
||||
legends: list[dict] = []
|
||||
rejected_legends: list[dict] = []
|
||||
assets: list[dict] = []
|
||||
unmatched_legends: list[dict] = []
|
||||
unmatched_assets: list[dict] = []
|
||||
matched_figures: list[dict] = []
|
||||
|
||||
for block in structured_blocks:
|
||||
if block.get("page_width"):
|
||||
page_width = float(block["page_width"])
|
||||
|
||||
for block in structured_blocks:
|
||||
role = block.get("role", "")
|
||||
if role == "figure_caption":
|
||||
if _is_body_mention(block):
|
||||
continue
|
||||
if not _is_formal_legend(block.get("text", ""), block, page_width):
|
||||
rejected_legends.append(block)
|
||||
continue
|
||||
legends.append(block)
|
||||
elif role == "figure_asset":
|
||||
assets.append(block)
|
||||
|
|
@ -187,6 +255,7 @@ def build_figure_inventory(structured_blocks: list[dict]) -> dict[str, Any]:
|
|||
"matched_figures": matched_figures,
|
||||
"unmatched_legends": unmatched_legends,
|
||||
"unmatched_assets": unmatched_assets,
|
||||
"rejected_legends": rejected_legends,
|
||||
"official_figure_count": len(matched_figures),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,17 +10,35 @@ from paperforge.core.io import write_json
|
|||
|
||||
def _clean_author_display(author_text: str) -> str:
|
||||
text = author_text
|
||||
text = re.sub(r'\$\s+\^', '$^', text)
|
||||
text = re.sub(r'\^\s+\{', '^{', text)
|
||||
text = re.sub(r'\}\s+\$', '}$', text)
|
||||
text = re.sub(r',(\s*)and(\s*)', r', and ', text)
|
||||
text = re.sub(r';(\s*)and(\s*)', r'; and ', text)
|
||||
text = re.sub(r'([,;:])\1+', r'\1', text)
|
||||
text = re.sub(r'\s{2,}', ' ', text)
|
||||
text = text.strip().strip(';, ')
|
||||
text = re.sub(r"\$\s+\^", "$^", text)
|
||||
text = re.sub(r"\^\s+\{", "^{", text)
|
||||
text = re.sub(r"\}\s+\$", "}$", text)
|
||||
text = re.sub(r",(\s*)and(\s*)", r", and ", text)
|
||||
text = re.sub(r";(\s*)and(\s*)", r"; and ", text)
|
||||
text = re.sub(r"([,;:])\1+", r"\1", text)
|
||||
text = re.sub(r"\s{2,}", " ", text)
|
||||
text = text.strip().strip(";, ")
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _name_likeness_score(text: str) -> int:
|
||||
"""Score how much the text looks like a human-name list vs affiliation/noise."""
|
||||
if not text:
|
||||
return 0
|
||||
score = 0
|
||||
if text.count(",") >= 1:
|
||||
score += text.count(",")
|
||||
if bool(re.search(r"[A-Z][a-z]+,\s+[A-Z]", text)):
|
||||
score += 3
|
||||
if bool(re.search(r"\band\b\s+[A-Z]", text)):
|
||||
score += 2
|
||||
if bool(re.search(r"[\*†‡§¶#]", text)):
|
||||
score += 1
|
||||
if any(kw in text.lower() for kw in ["university", "department", "institute", "college", "school of"]):
|
||||
score -= 5
|
||||
return score
|
||||
|
||||
|
||||
def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, Any]:
|
||||
candidates: dict[str, Any] = {
|
||||
"title": None,
|
||||
|
|
@ -47,7 +65,13 @@ def extract_frontmatter_candidates(blocks_structured_path: Path) -> dict[str, An
|
|||
if role == "paper_title":
|
||||
candidates["title"] = text
|
||||
elif role == "authors":
|
||||
candidates["authors_text"] = text
|
||||
if candidates["authors_text"] is None:
|
||||
candidates["authors_text"] = text
|
||||
else:
|
||||
current_score = _name_likeness_score(candidates["authors_text"])
|
||||
new_score = _name_likeness_score(text)
|
||||
if new_score > current_score:
|
||||
candidates["authors_text"] = text
|
||||
elif role in ("affiliation",):
|
||||
if "affiliation_blocks" not in candidates:
|
||||
candidates["affiliation_blocks"] = []
|
||||
|
|
@ -114,6 +138,12 @@ def resolve_metadata(
|
|||
# --- authors ---
|
||||
zotero_authors = source_metadata.get("authors", [])
|
||||
ocr_authors_text = frontmatter_candidates.get("authors_text", "")
|
||||
|
||||
_INST_KEYWORDS = ["university", "department", "institute", "college", "school of"]
|
||||
|
||||
def _is_affiliation_like(t: str) -> bool:
|
||||
return any(kw in t.lower() for kw in _INST_KEYWORDS)
|
||||
|
||||
ocr_author_list = (
|
||||
[a.strip() for a in re.split(r",\s+(?=[A-Z])", ocr_authors_text) if a.strip()] if ocr_authors_text else []
|
||||
)
|
||||
|
|
@ -124,7 +154,7 @@ def resolve_metadata(
|
|||
"source": "zotero",
|
||||
"confidence": 0.99,
|
||||
}
|
||||
elif ocr_author_list:
|
||||
elif ocr_author_list and not _is_affiliation_like(ocr_authors_text):
|
||||
resolved["authors"] = {
|
||||
"value": ocr_author_list,
|
||||
"source": "ocr_frontmatter",
|
||||
|
|
@ -140,7 +170,7 @@ def resolve_metadata(
|
|||
# --- authors_display (cleaned string for UI) ---
|
||||
if isinstance(zotero_authors, list) and len(zotero_authors) > 0:
|
||||
resolved["authors_display"] = ", ".join(zotero_authors)
|
||||
elif ocr_authors_text:
|
||||
elif ocr_authors_text and not _is_affiliation_like(ocr_authors_text):
|
||||
resolved["authors_display"] = _clean_author_display(ocr_authors_text)
|
||||
else:
|
||||
resolved["authors_display"] = ""
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ def _is_bogus_heading(text: str) -> bool:
|
|||
|
||||
_TAIL_ROLES = frozenset(
|
||||
{
|
||||
"backmatter_boundary_heading",
|
||||
"backmatter_heading",
|
||||
"backmatter_body",
|
||||
"tail_candidate_body",
|
||||
|
|
@ -206,19 +207,37 @@ def _extract_style_profile(block: dict) -> dict | None:
|
|||
|
||||
|
||||
def _build_heading_style_profiles(blocks: list[dict]) -> dict:
|
||||
heading_roles = frozenset({"section_heading", "subsection_heading", "backmatter_heading", "reference_heading"})
|
||||
profiles_with_sizes = []
|
||||
heading_roles = frozenset(
|
||||
{
|
||||
"backmatter_boundary_heading",
|
||||
"section_heading",
|
||||
"subsection_heading",
|
||||
"sub_subsection_heading",
|
||||
"backmatter_heading",
|
||||
"reference_heading",
|
||||
}
|
||||
)
|
||||
heading_items = []
|
||||
|
||||
for block in blocks:
|
||||
if block.get("role") in heading_roles:
|
||||
profile = _extract_style_profile(block)
|
||||
if profile is not None:
|
||||
profiles_with_sizes.append(profile["mean_size"])
|
||||
heading_items.append(
|
||||
{
|
||||
"max_size": profile["max_size"],
|
||||
"mean_size": profile["mean_size"],
|
||||
"profile": profile,
|
||||
"block": block,
|
||||
}
|
||||
)
|
||||
|
||||
if len(profiles_with_sizes) < 3:
|
||||
if len(heading_items) < 3:
|
||||
return {}
|
||||
|
||||
unique_sizes = sorted(set(profiles_with_sizes))
|
||||
# Use max_size for clustering — section headings often have slightly larger
|
||||
# first characters, making max_size more discriminating than mean.
|
||||
unique_sizes = sorted(set(item["max_size"] for item in heading_items))
|
||||
if not unique_sizes:
|
||||
return {}
|
||||
|
||||
|
|
@ -232,33 +251,70 @@ def _build_heading_style_profiles(blocks: list[dict]) -> dict:
|
|||
current = [s]
|
||||
clusters.append(current)
|
||||
|
||||
# Build full profiles for each cluster
|
||||
cluster_profiles = []
|
||||
for cluster in clusters:
|
||||
matching = []
|
||||
for block in blocks:
|
||||
if block.get("role") in heading_roles:
|
||||
p = _extract_style_profile(block)
|
||||
if p is not None and any(abs(p["mean_size"] - s) <= 2 for s in cluster):
|
||||
matching.append(p)
|
||||
cluster_profiles.append(matching)
|
||||
# Build full profile data for each cluster
|
||||
cluster_items = []
|
||||
for cluster_sizes in clusters:
|
||||
matching = [item for item in heading_items if any(abs(item["max_size"] - s) <= 2 for s in cluster_sizes)]
|
||||
cluster_items.append(matching)
|
||||
|
||||
clusters.sort(key=lambda c: sum(c) / len(c), reverse=True)
|
||||
# Sort clusters by max_size descending to establish hierarchy level
|
||||
clusters.sort(key=lambda c: max(c), reverse=True)
|
||||
cluster_items.sort(key=lambda items: max(i["max_size"] for i in items), reverse=True)
|
||||
|
||||
keys = ["primary", "subsection", "backmatter", "body"]
|
||||
keys = ["primary", "subsection", "sub_subsection", "backmatter", "body"]
|
||||
result = {}
|
||||
for i, (cluster, profiles) in enumerate(zip(clusters, cluster_profiles, strict=False)):
|
||||
for i, (cluster_sizes, items) in enumerate(zip(clusters, cluster_items, strict=False)):
|
||||
if i >= len(keys):
|
||||
break
|
||||
|
||||
profiles = [item["profile"] for item in items]
|
||||
|
||||
is_bold = any(p["is_bold"] for p in profiles)
|
||||
fonts = set()
|
||||
for p in profiles:
|
||||
fonts.update(p["font_families"])
|
||||
|
||||
# Compute spacing before/after by searching ALL blocks on the same page
|
||||
space_before_vals = []
|
||||
space_after_vals = []
|
||||
for item in items:
|
||||
heading_block = item["block"]
|
||||
heading_page = heading_block.get("page")
|
||||
heading_bbox = heading_block.get("bbox") or heading_block.get("block_bbox")
|
||||
if heading_bbox and len(heading_bbox) >= 4 and heading_page is not None:
|
||||
h_y1 = heading_bbox[1]
|
||||
h_y2 = heading_bbox[3]
|
||||
|
||||
nearest_above_bottom = None
|
||||
nearest_below_top = None
|
||||
for other in blocks:
|
||||
if other is heading_block:
|
||||
continue
|
||||
if other.get("page") == heading_page:
|
||||
obbox = other.get("bbox") or other.get("block_bbox")
|
||||
if obbox and len(obbox) >= 4:
|
||||
o_y2 = obbox[3]
|
||||
o_y1 = obbox[1]
|
||||
if o_y2 <= h_y1 and (nearest_above_bottom is None or o_y2 > nearest_above_bottom):
|
||||
nearest_above_bottom = o_y2
|
||||
if o_y1 >= h_y2 and (nearest_below_top is None or o_y1 < nearest_below_top):
|
||||
nearest_below_top = o_y1
|
||||
|
||||
if nearest_above_bottom is not None:
|
||||
space_before_vals.append(h_y1 - nearest_above_bottom)
|
||||
if nearest_below_top is not None:
|
||||
space_after_vals.append(nearest_below_top - h_y2)
|
||||
|
||||
space_before = sum(space_before_vals) / len(space_before_vals) if space_before_vals else 0.0
|
||||
space_after = sum(space_after_vals) / len(space_after_vals) if space_after_vals else 0.0
|
||||
|
||||
result[keys[i]] = {
|
||||
"size_min": min(cluster),
|
||||
"size_max": max(cluster),
|
||||
"size_min": min(cluster_sizes),
|
||||
"size_max": max(cluster_sizes),
|
||||
"bold": is_bold,
|
||||
"fonts": fonts,
|
||||
"space_before": space_before,
|
||||
"space_after": space_after,
|
||||
}
|
||||
|
||||
return result
|
||||
|
|
@ -271,21 +327,24 @@ def _disambiguate_heading_role(block: dict, style_profiles: dict) -> str | None:
|
|||
|
||||
size = profile["mean_size"]
|
||||
|
||||
for role_key in ("primary", "subsection", "backmatter", "body"):
|
||||
for role_key, role_name in [
|
||||
("primary", "section_heading"),
|
||||
("subsection", "subsection_heading"),
|
||||
("sub_subsection", "sub_subsection_heading"),
|
||||
("backmatter", "backmatter_heading"),
|
||||
("body", None),
|
||||
]:
|
||||
cfg = style_profiles.get(role_key)
|
||||
if cfg is None:
|
||||
continue
|
||||
if cfg["size_min"] <= size <= cfg["size_max"]:
|
||||
if role_key == "primary":
|
||||
return "section_heading"
|
||||
elif role_key == "subsection":
|
||||
return "subsection_heading"
|
||||
elif role_key == "backmatter":
|
||||
if role_key == "backmatter":
|
||||
if profile["is_bold"]:
|
||||
return "backmatter_heading"
|
||||
return role_name
|
||||
return None
|
||||
elif role_key == "body":
|
||||
return None
|
||||
return role_name
|
||||
|
||||
return None
|
||||
|
||||
|
|
@ -335,7 +394,7 @@ def _reorder_tail_run(
|
|||
body_pool: list[dict] = []
|
||||
for block in tail_blocks:
|
||||
role = block.get("role")
|
||||
if role == "backmatter_heading":
|
||||
if role in ("backmatter_boundary_heading", "backmatter_heading"):
|
||||
backmatter_sections.append({"heading": block, "bodies": []})
|
||||
elif role == "reference_heading":
|
||||
ref_section = {"heading": block, "bodies": []}
|
||||
|
|
@ -407,11 +466,16 @@ def _reorder_tail_run(
|
|||
else:
|
||||
orphan_blocks.append(body)
|
||||
|
||||
# Phase 5 — emit: non-tail pass (body/unknown), then backmatter, then references
|
||||
# Phase 5 — emit: non-tail pass, then boundary heading first
|
||||
# (backmatter container), then sub-sections, then references
|
||||
result: list[dict] = []
|
||||
result.extend(non_tail_pass)
|
||||
result.extend(carried_bodies)
|
||||
for sec in backmatter_sections:
|
||||
boundary_secs = [s for s in backmatter_sections
|
||||
if s["heading"].get("role") == "backmatter_boundary_heading"]
|
||||
sub_secs = [s for s in backmatter_sections
|
||||
if s["heading"].get("role") != "backmatter_boundary_heading"]
|
||||
for sec in boundary_secs + sub_secs:
|
||||
result.append(sec["heading"])
|
||||
result.extend(sec["bodies"])
|
||||
if ref_section is not None and ref_section is not carried_ref:
|
||||
|
|
@ -442,7 +506,7 @@ def _reorder_tail_run_fifo(
|
|||
|
||||
for block in tail_blocks:
|
||||
role = block.get("role")
|
||||
if role == "backmatter_heading":
|
||||
if role in ("backmatter_boundary_heading", "backmatter_heading"):
|
||||
sec = {"heading": block, "bodies": []}
|
||||
backmatter_sections.append(sec)
|
||||
heading_queue.append(sec)
|
||||
|
|
@ -471,7 +535,11 @@ def _reorder_tail_run_fifo(
|
|||
|
||||
result: list[dict] = []
|
||||
result.extend(non_tail_pass)
|
||||
for sec in backmatter_sections:
|
||||
boundary_secs = [s for s in backmatter_sections
|
||||
if s["heading"].get("role") == "backmatter_boundary_heading"]
|
||||
sub_secs = [s for s in backmatter_sections
|
||||
if s["heading"].get("role") != "backmatter_boundary_heading"]
|
||||
for sec in boundary_secs + sub_secs:
|
||||
result.append(sec["heading"])
|
||||
result.extend(sec["bodies"])
|
||||
if ref_section is not None and ref_section is not carried_ref:
|
||||
|
|
@ -511,7 +579,9 @@ def _promote_tail_body_candidates(
|
|||
continue
|
||||
|
||||
page_blocks = [result[i] for i in indices]
|
||||
local_headings = [b for b in page_blocks if b.get("role") == "backmatter_heading"]
|
||||
local_headings = [
|
||||
b for b in page_blocks if b.get("role") in ("backmatter_heading", "backmatter_boundary_heading")
|
||||
]
|
||||
ref_heading = next((b for b in page_blocks if b.get("role") == "reference_heading"), None)
|
||||
local_anchors = local_headings
|
||||
local_tops = []
|
||||
|
|
@ -634,7 +704,7 @@ def _detect_forward_body_end(blocks: list[dict]) -> int | None:
|
|||
|
||||
for page in pages:
|
||||
roles = {b.get("role") for b in by_page[page]}
|
||||
has_body = bool(roles & {"body_paragraph", "section_heading", "subsection_heading"})
|
||||
has_body = bool(roles & {"body_paragraph", "section_heading", "subsection_heading", "sub_subsection_heading"})
|
||||
has_tail = bool(roles & _TAIL_ROLES)
|
||||
|
||||
if has_body and not has_tail:
|
||||
|
|
@ -674,7 +744,7 @@ def _detect_backward_backmatter_start(blocks: list[dict]) -> int | None:
|
|||
page_blocks = by_page[page]
|
||||
roles = {b.get("role") for b in page_blocks}
|
||||
|
||||
if "reference_heading" in roles or "backmatter_heading" in roles:
|
||||
if "reference_heading" in roles or "backmatter_heading" in roles or "backmatter_boundary_heading" in roles:
|
||||
return page
|
||||
|
||||
dense_refs = sum(1 for b in page_blocks if b.get("role") == "reference_item")
|
||||
|
|
@ -731,7 +801,8 @@ def _assign_tail_spread_ownership(
|
|||
with _spread_anchor so the page-local reorder pass does not reassign them.
|
||||
Unanchored candidates inside the spread stay as tail_candidate_body.
|
||||
"""
|
||||
anchors = [b for b in blocks if b.get("role") == "backmatter_heading"]
|
||||
tail_heading_roles = {"backmatter_heading", "backmatter_boundary_heading"}
|
||||
anchors = [b for b in blocks if b.get("role") in tail_heading_roles]
|
||||
ref_heading = next((b for b in blocks if b.get("role") == "reference_heading"), None)
|
||||
|
||||
if tail_spread is not None:
|
||||
|
|
@ -1007,7 +1078,7 @@ def render_fulltext_markdown(
|
|||
lines.append("")
|
||||
emitted_pages.add(block_page)
|
||||
|
||||
if role == "backmatter_heading" or role == "reference_heading":
|
||||
if role == "backmatter_boundary_heading" or role == "backmatter_heading" or role == "reference_heading":
|
||||
lines.append(f"## {text}")
|
||||
lines.append("")
|
||||
elif role == "section_heading":
|
||||
|
|
@ -1020,7 +1091,7 @@ def render_fulltext_markdown(
|
|||
else:
|
||||
lines.append(f"## {text}")
|
||||
lines.append("")
|
||||
elif role == "subsection_heading":
|
||||
elif role == "subsection_heading" or role == "sub_subsection_heading":
|
||||
lines.append(f"### {text}")
|
||||
lines.append("")
|
||||
elif role in ("backmatter_body", "tail_candidate_body", "body_paragraph"):
|
||||
|
|
|
|||
|
|
@ -109,6 +109,226 @@ def _looks_like_reference(text: str) -> bool:
|
|||
return bool(_REFERENCE_PATTERN.match(text.strip()))
|
||||
|
||||
|
||||
def _is_backmatter_boundary_heading(block: dict, page_blocks: list[dict]) -> bool:
|
||||
text = str(block.get("block_content", "") or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
page_num = block.get("page", 1) or 1
|
||||
if page_num < 8:
|
||||
return False
|
||||
|
||||
raw_label = str(block.get("block_label", "") or "").strip()
|
||||
|
||||
is_heading_label = raw_label == "paragraph_title"
|
||||
span_meta = block.get("span_metadata", {}) or {}
|
||||
is_visually_heading = False
|
||||
if isinstance(span_meta, dict):
|
||||
font_size = span_meta.get("size", 0) or 0
|
||||
font_flags = (span_meta.get("flags", "") or "").lower()
|
||||
is_visually_heading = ("bold" in font_flags and font_size >= 11) or font_size >= 14
|
||||
elif isinstance(span_meta, list):
|
||||
sizes = [s.get("size", 0) or 0 for s in span_meta if isinstance(s, dict)]
|
||||
bold_flags = any(s.get("flags", 0) & 16 for s in span_meta if isinstance(s, dict))
|
||||
avg_size = sum(sizes) / len(sizes) if sizes else 0
|
||||
is_visually_heading = bold_flags or avg_size >= 14
|
||||
|
||||
if not (is_heading_label or is_visually_heading):
|
||||
return False
|
||||
|
||||
upper = text.upper()
|
||||
has_container_words = (
|
||||
"ADDITIONAL" in upper or "SUPPLEMENTARY" in upper or "DECLARATION" in upper or "INFORMATION" in upper
|
||||
)
|
||||
if not has_container_words:
|
||||
return False
|
||||
|
||||
if len(text) <= 20:
|
||||
return False
|
||||
|
||||
lower = text.lower()
|
||||
if lower in ("references", "bibliography"):
|
||||
return False
|
||||
if lower in _BACKMATTER_HEADINGS:
|
||||
return False
|
||||
return lower not in _BACKMATTER_TITLE_DENY_LIST
|
||||
|
||||
|
||||
def _looks_like_author_list(text: str) -> bool:
|
||||
"""Check if text looks like a list of author names (not a title or body)."""
|
||||
if not text:
|
||||
return False
|
||||
has_name_comma = bool(re.search(r"[A-Z][a-z]+,\s+[A-Z]", text))
|
||||
has_and_name = bool(re.search(r"\band\b\s+[A-Z][a-z]+", text))
|
||||
has_author_marker = bool(re.search(r"[\*†‡§¶#]", text))
|
||||
has_two_name_pairs = bool(re.search(r"[A-Z][a-z]+\s+[A-Z][a-z]+\s*[·•,;]\s*[A-Z][a-z]+", text))
|
||||
has_et_al = "et al" in text.lower()
|
||||
return (has_name_comma or has_and_name or has_author_marker or has_two_name_pairs or has_et_al) and len(text) < 500
|
||||
|
||||
|
||||
def _looks_like_affiliation(text: str) -> bool:
|
||||
"""Check if text looks like an affiliation block."""
|
||||
lower_txt = text.lower()
|
||||
inst_keywords = [
|
||||
"university",
|
||||
"department",
|
||||
"institute",
|
||||
"college",
|
||||
"school of",
|
||||
"faculty",
|
||||
"laboratory",
|
||||
"lab",
|
||||
"hospital",
|
||||
"center",
|
||||
"centre",
|
||||
"academy",
|
||||
"division",
|
||||
"initiative",
|
||||
"regenerative medicine",
|
||||
"research",
|
||||
"science",
|
||||
"technology",
|
||||
"medicine",
|
||||
"school of materials",
|
||||
]
|
||||
has_inst = any(kw in lower_txt for kw in inst_keywords)
|
||||
has_city_country = bool(
|
||||
re.search(
|
||||
r"(?:,\s*)(?:USA|UK|China|Germany|France|Japan|Italy|Canada|"
|
||||
r"Australia|Brazil|India|Korea|Spain|Netherlands|Switzerland|"
|
||||
r"Sweden|Norway|Denmark|Austria|Belgium|Finland|Poland|"
|
||||
r"Russia|Mexico|Argentina|Singapore|Taiwan|Hong\s*Kong)",
|
||||
text,
|
||||
)
|
||||
)
|
||||
has_number_prefix = bool(re.match(r"^[\$\s]*\^?\d+\s*", text))
|
||||
has_superscript_number = bool(re.search(r"\$?\^\d+\$?", text))
|
||||
return has_inst or (has_city_country and has_number_prefix) or has_superscript_number
|
||||
|
||||
|
||||
def _infer_heading_level(text: str, font_size: float = 0) -> str:
|
||||
"""Infer heading level for unnumbered papers using text content heuristics.
|
||||
|
||||
Uses case patterns, word count, and optional font size to distinguish
|
||||
top-level section headings from lower-level sub-sections. Font size
|
||||
overrides content heuristics when >= 14pt (clearly a major heading).
|
||||
"""
|
||||
if not text:
|
||||
return "section_heading"
|
||||
if font_size >= 14:
|
||||
return "section_heading"
|
||||
upper_ratio = sum(1 for c in text if c.isupper()) / max(len(text), 1)
|
||||
is_mostly_upper = upper_ratio > 0.7
|
||||
sentence_verbs = {" is ", " are ", " was ", " were ", " have ", " has ", " been "}
|
||||
has_sentence_verb = any(v in text.lower() for v in sentence_verbs)
|
||||
word_count = len(text.split())
|
||||
if is_mostly_upper and word_count >= 1 and not has_sentence_verb:
|
||||
return "section_heading"
|
||||
if word_count >= 2 and not has_sentence_verb:
|
||||
return "subsection_heading"
|
||||
if word_count == 1 and not has_sentence_verb:
|
||||
return "sub_subsection_heading"
|
||||
return "section_heading"
|
||||
|
||||
|
||||
def _detect_frontmatter_zone(
|
||||
block: dict,
|
||||
page_blocks: list[dict],
|
||||
page_height: float,
|
||||
page_width: float,
|
||||
style_profiles: dict | None = None,
|
||||
) -> str | None:
|
||||
"""Detect frontmatter zone for a block on page 1.
|
||||
|
||||
Returns one of: ``title_zone``, ``author_zone``, ``affiliation_zone``,
|
||||
``journal_furniture_zone``, ``abstract_zone``, or ``None``.
|
||||
"""
|
||||
page_num = block.get("page", 1) or 1
|
||||
if page_num > 1:
|
||||
return None
|
||||
|
||||
bbox = block.get("block_bbox", [0, 0, 0, 0])
|
||||
if len(bbox) < 4:
|
||||
return None
|
||||
|
||||
text = str(block.get("block_content", "") or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
lower_txt = text.lower()
|
||||
raw_label = str(block.get("block_label", "") or "").strip()
|
||||
x1, y1, x2 = bbox[0], bbox[1], bbox[2]
|
||||
|
||||
# --- abstract_zone ---
|
||||
if lower_txt.startswith("abstract") and len(text) < 30:
|
||||
return "abstract_zone"
|
||||
|
||||
# --- journal_furniture_zone ---
|
||||
furniture_signals = [
|
||||
"submitted",
|
||||
"accepted",
|
||||
"published",
|
||||
"received",
|
||||
"copyright",
|
||||
"©",
|
||||
"doi:",
|
||||
"doi ",
|
||||
"https://doi.org",
|
||||
"academic editor",
|
||||
"how to cite",
|
||||
"to cite this article",
|
||||
"creative commons",
|
||||
"cc by",
|
||||
"cc license",
|
||||
"this is an open-access article",
|
||||
"reviewed by",
|
||||
"edited by",
|
||||
"present address",
|
||||
]
|
||||
if any(s in lower_txt for s in furniture_signals):
|
||||
return "journal_furniture_zone"
|
||||
|
||||
narrow_furniture = [
|
||||
"citation:",
|
||||
"correspondence",
|
||||
"orcid",
|
||||
"these authors contributed equally",
|
||||
"equal contribution",
|
||||
"additional information",
|
||||
]
|
||||
if any(s in lower_txt for s in narrow_furniture):
|
||||
block_width = x2 - x1
|
||||
is_narrow = page_width > 0 and block_width < page_width * 0.35
|
||||
is_top_half = page_height > 0 and y1 < page_height * 0.5
|
||||
if is_narrow or is_top_half:
|
||||
return "journal_furniture_zone"
|
||||
|
||||
# --- title_zone ---
|
||||
if page_height > 0 and y1 < page_height * 0.2:
|
||||
block_width = x2 - x1
|
||||
is_wide_enough = page_width <= 0 or block_width > page_width * 0.4
|
||||
if is_wide_enough and lower_txt not in _BACKMATTER_TITLE_DENY_LIST and not _looks_like_author_list(text):
|
||||
if raw_label in ("paragraph_title", "doc_title"):
|
||||
return "title_zone"
|
||||
if raw_label == "text" and len(text) < 80:
|
||||
return "title_zone"
|
||||
|
||||
# --- author_zone ---
|
||||
if (
|
||||
page_height > 0
|
||||
and y1 < page_height * 0.4
|
||||
and _looks_like_author_list(text)
|
||||
and not _looks_like_affiliation(text)
|
||||
):
|
||||
return "author_zone"
|
||||
|
||||
# --- affiliation_zone ---
|
||||
if page_height > 0 and y1 < page_height * 0.6 and _looks_like_affiliation(text):
|
||||
return "affiliation_zone"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def assign_block_role(
|
||||
block: dict,
|
||||
page_blocks: list[dict],
|
||||
|
|
@ -147,6 +367,50 @@ def assign_block_role(
|
|||
evidence=[f"table prefix matched: {text[:60]}"],
|
||||
)
|
||||
|
||||
# --- Page-1 frontmatter zone pre-filter ---
|
||||
page_num = block.get("page", 1) or 1
|
||||
zone = None
|
||||
if page_num == 1 and raw_label in ("paragraph_title", "text"):
|
||||
zone = _detect_frontmatter_zone(
|
||||
block,
|
||||
page_blocks,
|
||||
page_height,
|
||||
page_width,
|
||||
style_profiles,
|
||||
)
|
||||
|
||||
if zone == "title_zone":
|
||||
lower_stripped = text.strip().lstrip("*•·-–—").lower()
|
||||
if lower_stripped not in _BACKMATTER_TITLE_DENY_LIST:
|
||||
return RoleAssignment(
|
||||
role="paper_title",
|
||||
confidence=0.8,
|
||||
evidence=[f"page-1 zone title_zone: {text[:60]}"],
|
||||
)
|
||||
|
||||
if zone == "author_zone":
|
||||
return RoleAssignment(
|
||||
role="authors",
|
||||
confidence=0.8,
|
||||
evidence=[f"page-1 zone author_zone: {text[:60]}"],
|
||||
)
|
||||
|
||||
if zone == "affiliation_zone":
|
||||
return RoleAssignment(
|
||||
role="affiliation",
|
||||
confidence=0.8,
|
||||
evidence=[f"page-1 zone affiliation_zone: {text[:60]}"],
|
||||
)
|
||||
|
||||
if zone == "journal_furniture_zone":
|
||||
return RoleAssignment(
|
||||
role="frontmatter_noise",
|
||||
confidence=0.8,
|
||||
evidence=[f"page-1 zone journal_furniture_zone: {text[:60]}"],
|
||||
)
|
||||
|
||||
# zone == "abstract_zone" → let existing logic handle
|
||||
|
||||
# Paddle priors
|
||||
if raw_label == "paragraph_title":
|
||||
stripped = text.strip().lstrip("*•·-–—")
|
||||
|
|
@ -187,6 +451,14 @@ def assign_block_role(
|
|||
confidence=0.5,
|
||||
evidence=[f"backmatter heading text on page 1, treated as section: {text[:60]}"],
|
||||
)
|
||||
# Backmatter boundary container heading (e.g. "ADDITIONAL INFORMATION AND DECLARATIONS")
|
||||
# Checked after specific heading detection but before numbered/generic fallback.
|
||||
if _is_backmatter_boundary_heading(block, page_blocks):
|
||||
return RoleAssignment(
|
||||
role="backmatter_boundary_heading",
|
||||
confidence=0.7,
|
||||
evidence=[f"backmatter boundary heading: {text[:60]}"],
|
||||
)
|
||||
if _has_heading_numbering(text):
|
||||
return RoleAssignment(
|
||||
role="section_heading" if re.match(r"^\d+\s", text) else "subsection_heading",
|
||||
|
|
@ -199,7 +471,7 @@ def assign_block_role(
|
|||
if bbox[1] < max(page_height, 1) * 0.25:
|
||||
if lower in _BACKMATTER_TITLE_DENY_LIST:
|
||||
return RoleAssignment(
|
||||
role="section_heading",
|
||||
role="section_heading" if re.match(r"^\d+\s", text) else "subsection_heading",
|
||||
confidence=0.5,
|
||||
evidence=[f"backmatter title in title zone, treated as heading: {text[:60]}"],
|
||||
)
|
||||
|
|
@ -213,10 +485,11 @@ def assign_block_role(
|
|||
confidence=0.5,
|
||||
evidence=[f"unnumbered paragraph_title on page 1 outside title zone: {text[:60]}"],
|
||||
)
|
||||
level = _infer_heading_level(text)
|
||||
return RoleAssignment(
|
||||
role="section_heading",
|
||||
role=level,
|
||||
confidence=0.6,
|
||||
evidence=[f"unnumbered paragraph_title on page {page_num}, treated as heading: {text[:60]}"],
|
||||
evidence=[f"unnumbered paragraph_title, inferred level {level}: {text[:60]}"],
|
||||
)
|
||||
|
||||
if raw_label == "figure_title":
|
||||
|
|
@ -401,7 +674,7 @@ def assign_block_role(
|
|||
no_sentence_verbs = not (len(text) > 50 and any(v in text.lower() for v in sentence_verbs))
|
||||
if in_top_80 and wide_enough and no_sentence_verbs:
|
||||
return RoleAssignment(
|
||||
role="section_heading" if re.match(r"^\d+\s", text) else "subsection_heading",
|
||||
role="section_heading",
|
||||
confidence=0.65,
|
||||
evidence=[f"numbered text block: {text[:60]}"],
|
||||
)
|
||||
|
|
@ -413,7 +686,7 @@ def assign_block_role(
|
|||
evidence=[f"references heading from text block: {text[:40]}"],
|
||||
)
|
||||
|
||||
# Style-aware heading disambiguation (Task 5)
|
||||
# Style-aware heading disambiguation — primary signal for unnumbered papers
|
||||
if style_profiles is not None and style_profiles:
|
||||
from paperforge.worker.ocr_render import _disambiguate_heading_role
|
||||
|
||||
|
|
@ -424,7 +697,7 @@ def assign_block_role(
|
|||
if in_top_80 and len(text) >= 5 and len(text) < 60 and ". " not in text:
|
||||
return RoleAssignment(
|
||||
role=style_suggested,
|
||||
confidence=0.65,
|
||||
confidence=0.7,
|
||||
evidence=[f"style-aware heading detection: role={style_suggested}, text={text[:40]}"],
|
||||
)
|
||||
|
||||
|
|
@ -441,12 +714,21 @@ def assign_block_role(
|
|||
bbox = block.get("block_bbox", [0, 0, 0, 0])
|
||||
in_top_80 = page_height == 0 or (len(bbox) >= 4 and bbox[1] < page_height * 0.8)
|
||||
if in_top_80:
|
||||
heading_level = _infer_heading_level(text, font_size)
|
||||
return RoleAssignment(
|
||||
role="section_heading",
|
||||
role=heading_level,
|
||||
confidence=0.65,
|
||||
evidence=[f"heading-style text block: size={font_size}, flags={font_flags}, text={text[:40]}"],
|
||||
evidence=[f"heading-style text block: size={font_size}, flags={font_flags}, level={heading_level}, text={text[:40]}"],
|
||||
)
|
||||
|
||||
# Backmatter boundary container heading detection for text blocks
|
||||
if _is_backmatter_boundary_heading(block, page_blocks):
|
||||
return RoleAssignment(
|
||||
role="backmatter_boundary_heading",
|
||||
confidence=0.7,
|
||||
evidence=[f"backmatter boundary heading from text block: {text[:60]}"],
|
||||
)
|
||||
|
||||
if len(text) < 20:
|
||||
return RoleAssignment(
|
||||
role="unknown_structural",
|
||||
|
|
|
|||
Loading…
Reference in a new issue