From 924fcfd54fd9acf0957c9b19b7343abaafb68e54 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Mon, 22 Jun 2026 00:46:54 +0800 Subject: [PATCH] fix: footnote blocks emit after refs; frontmatter_noise renders on backmatter pages; visual heading grading; figure inner-text crop exclusion --- paperforge/worker/ocr_figures.py | 15 +++++++++++++++ paperforge/worker/ocr_render.py | 29 +++++++++++++++++++++-------- paperforge/worker/ocr_roles.py | 20 +++++++++++++++++--- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/paperforge/worker/ocr_figures.py b/paperforge/worker/ocr_figures.py index 45e92baa..f44b4370 100644 --- a/paperforge/worker/ocr_figures.py +++ b/paperforge/worker/ocr_figures.py @@ -1501,6 +1501,21 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12 entry["bridge_block_ids"] = [str(b.get("block_id", "")) for b in local_bridges if b.get("block_id")] if len(matched_assets) > 1: entry["cluster_bbox"] = _cluster_bbox([a.get("bbox", [0, 0, 0, 0]) for a in matched_assets]) + # Push cluster_bbox top down past figure_inner_text blocks (panel labels) + # to exclude (A), (B) etc. from the cropped figure image. + leg_page = int(legend_page or 0) + for ib in structured_blocks: + if int(ib.get("page", 0) or 0) != leg_page or ib.get("role") != "figure_inner_text": + continue + ib_bbox = ib.get("bbox") or ib.get("block_bbox") or [0, 0, 0, 0] + cb = entry["cluster_bbox"] + if len(ib_bbox) >= 4 and len(cb) >= 4: + # Horizontal overlap: inner text sits in the figure's column + x_overlap = ib_bbox[0] < cb[2] and ib_bbox[2] > cb[0] + # Text starts above cluster top and extends into it + is_above = ib_bbox[1] < cb[1] and ib_bbox[3] > cb[1] + if x_overlap and is_above: + cb[1] = max(cb[1], ib_bbox[3]) matched_figures.append(entry) for _i, asset in enumerate(assets): diff --git a/paperforge/worker/ocr_render.py b/paperforge/worker/ocr_render.py index 4f931a1c..db392d79 100644 --- a/paperforge/worker/ocr_render.py +++ b/paperforge/worker/ocr_render.py @@ -381,9 +381,10 @@ def _reorder_tail_run( # Column-sort is maintained; ref items are grouped at the end. if skip_section_grouping: ref_roles = frozenset({"reference_heading", "reference_item", "reference_body"}) - non_ref = [b for b in tail_blocks if b.get("role") not in ref_roles] + non_ref = [b for b in tail_blocks if b.get("role") not in ref_roles and b.get("role") != "footnote"] refs = [b for b in tail_blocks if b.get("role") in ref_roles] - return non_ref + refs, carried_ref, carried_backmatter + fnotes = [b for b in tail_blocks if b.get("role") == "footnote"] + return non_ref + refs + fnotes, carried_ref, carried_backmatter # Quick check: do blocks have bbox data? If not, use FIFO fallback. has_geo = any( @@ -400,6 +401,7 @@ def _reorder_tail_run( ref_section: dict | None = carried_ref ref_items: list[dict] = [] non_tail_pass: list[dict] = [] + footnote_blocks: list[dict] = [] orphan_blocks: list[dict] = [] carried_bodies: list[dict] = [] @@ -430,6 +432,8 @@ def _reorder_tail_run( non_tail_pass.append(block) else: non_tail_pass.append(block) + elif role == "footnote": + footnote_blocks.append(block) else: non_tail_pass.append(block) @@ -505,9 +509,9 @@ def _reorder_tail_run( # Phase 5 — emit: non-tail pass (frontmatter), then backmatter # sections sorted by y-position (boundary-level and sub-headings - # emit in reading order), then reference sections. The y-sort is - # applied only to backmatter_sections to avoid carrying cross-page - # relationship artifacts from previous pages' carried blocks. + # emit in reading order), then reference sections, then footnotes. The + # y-sort is applied only to backmatter_sections to avoid carrying + # cross-page relationship artifacts from previous pages' carried blocks. result: list[dict] = [] result.extend(non_tail_pass) result.extend(carried_bodies) @@ -526,6 +530,7 @@ def _reorder_tail_run( if ref_section.get("heading"): result.append(ref_section["heading"]) result.extend(ref_section["bodies"]) + result.extend(footnote_blocks) result.extend(orphan_blocks) next_backmatter = carried_backmatter @@ -1237,7 +1242,9 @@ def render_fulltext_markdown( if role == "structured_insert": pass # render as callout below elif not block.get("render_default", True): - continue + bm_start = getattr(document_structure, "spread_start", None) if document_structure else None + if role != "frontmatter_noise" or block_page is None or not bm_start or block_page < bm_start: + continue if role in CONSUMED_FRONTMATTER_ROLES and int(block.get("page", 0) or 0) <= 2: continue _SKIPPED_BODY_ROLES = { @@ -1251,7 +1258,11 @@ def render_fulltext_markdown( "figure_inner_text", } if role in _SKIPPED_BODY_ROLES: - continue + bm_start = getattr(document_structure, "spread_start", None) if document_structure else None + if role == "frontmatter_noise" and block_page is not None and bm_start and block_page >= bm_start: + pass + else: + continue block_id = block.get("block_id") block_key = _page_block_key(block_page, block_id) @@ -1271,7 +1282,9 @@ def render_fulltext_markdown( if role == "structured_insert": pass # render as callout below elif not block.get("render_default", True): - continue + bm_start2 = getattr(document_structure, "spread_start", None) if document_structure else None + if role != "frontmatter_noise" or block_page is None or not bm_start2 or block_page < bm_start2: + continue if role == "backmatter_boundary_heading" or role == "backmatter_heading": last_structured_insert_page = None last_structured_insert_bbox = None diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index a99f09b6..754fcf72 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -442,7 +442,21 @@ def _infer_heading_level( if match["size_compatible"] and match["match_score"] > 0.6: return candidate_role - # Legacy heuristics (fallback when no profile data) + # Visual signal: font_size + bold overrides text heuristics + ss = (block or {}).get("span_signature") or {} + fs = ss.get("font_size_median", 0) or ss.get("font_size", 0) or font_size + is_bold = ss.get("bold", False) + if fs > 0: + # Larger font (section_heading tier) → highest level + if fs >= 11: + return "section_heading" + # Same-as-body bold → intermediate level + if is_bold: + return "subsection_heading" + # Same-as-body regular → lowest level + return "sub_subsection_heading" + + # Legacy text heuristics (fallback when no visual data) upper_ratio = sum(1 for c in text if c.isupper()) / max(len(text), 1) is_mostly_upper = upper_ratio > 0.7 sentence_verbs = {" is ", " are ", " was ", " were ", " have ", " has ", " been "} @@ -1211,7 +1225,7 @@ def assign_block_role( confidence=0.5, evidence=[f"unnumbered paragraph_title on page 1 outside title zone: {text[:60]}"], ) - level = _infer_heading_level(text) + level = _infer_heading_level(text, block=block) return RoleAssignment( role=level, confidence=0.6, @@ -1473,7 +1487,7 @@ def assign_block_role( bbox = block.get("block_bbox", [0, 0, 0, 0]) in_top_80 = page_height == 0 or (len(bbox) >= 4 and bbox[1] < page_height * 0.8) if in_top_80: - heading_level = _infer_heading_level(text, font_size) + heading_level = _infer_heading_level(text, font_size, block=block) return RoleAssignment( role=heading_level, confidence=0.65,