From c974ba74801566aeddfbbfd631e34600fe6a9af9 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Sat, 6 Jun 2026 13:28:46 +0800 Subject: [PATCH] fix: remove open-ended authors text heuristics, anchor to page-1 zone only --- _check_rebuild.py | 50 ++ ...r-author-anchor-and-nonbody-insert-plan.md | 439 ++++++++++++++++ ...06-ocr-structure-truth-convergence-plan.md | 480 ++++++++++++++++++ paperforge/worker/ocr_roles.py | 20 - tests/test_ocr_roles.py | 25 + 5 files changed, 994 insertions(+), 20 deletions(-) create mode 100644 _check_rebuild.py create mode 100644 docs/superpowers/plans/2026-06-06-ocr-author-anchor-and-nonbody-insert-plan.md create mode 100644 docs/superpowers/plans/2026-06-06-ocr-structure-truth-convergence-plan.md diff --git a/_check_rebuild.py b/_check_rebuild.py new file mode 100644 index 00000000..5ff13a5a --- /dev/null +++ b/_check_rebuild.py @@ -0,0 +1,50 @@ +from pathlib import Path +import json + +vault = Path(r"D:\L\OB\Literature-hub") +structured_path = vault / "System" / "PaperForge" / "ocr" / "SAN9AYVR" / "blocks.structured.jsonl" +profiles_path = vault / "System" / "PaperForge" / "ocr" / "SAN9AYVR" / "role_span_profiles.json" + +if structured_path.exists(): + print(f"structured blocks: {structured_path.stat().st_size} bytes") + with open(structured_path, encoding="utf-8") as f: + line = f.readline() + row = json.loads(line) + has_span = "span_metadata" in row + has_role = row.get("role") + has_conf = row.get("role_confidence") + print(f" first block role={has_role}, conf={has_conf}, has_span_metadata={has_span}") + + total = 0 + with_span = 0 + with open(structured_path, encoding="utf-8") as f: + for line in f: + total += 1 + row = json.loads(line) + if row.get("span_metadata"): + with_span += 1 + print(f" total blocks: {total}, with span_metadata: {with_span} ({100*with_span/total:.1f}%)") +else: + print("structured blocks not found") + +if profiles_path.exists(): + profiles = json.loads(profiles_path.read_text(encoding="utf-8")) + print(f"\nrole_span_profiles.json: {len(profiles)} roles") + for role, p in sorted(profiles.items()): + print(f" {role}: count={p['block_count']}, size={p['mean_size']}, quality={p['quality']}") +else: + print("role_span_profiles.json not found") + +second_pass_total = 0 +span_alt_total = 0 +with open(structured_path, encoding="utf-8") as f: + for line in f: + row = json.loads(line) + ev = row.get("evidence", []) + for e in ev: + if "second_pass" in e: + second_pass_total += 1 + if "span_alternatives" in e: + span_alt_total += 1 +print(f"\nSecond-pass applied: {second_pass_total} blocks") +print(f"Span alternatives flagged: {span_alt_total} blocks") diff --git a/docs/superpowers/plans/2026-06-06-ocr-author-anchor-and-nonbody-insert-plan.md b/docs/superpowers/plans/2026-06-06-ocr-author-anchor-and-nonbody-insert-plan.md new file mode 100644 index 00000000..4d4dd01e --- /dev/null +++ b/docs/superpowers/plans/2026-06-06-ocr-author-anchor-and-nonbody-insert-plan.md @@ -0,0 +1,439 @@ +# OCR Author Anchor And Non-Body Insert Plan + +Date: 2026-06-06 +Scope: `ocr_roles.py`, `ocr_document.py`, `ocr_metadata.py`, `ocr_blocks.py`, related tests, real-paper validation on `SAN9AYVR` + +## Goal + +收掉当前最不稳的一条链: + +1. 开放式 `authors` 文本识别会误伤正文 +2. early-page 作者简介 / profile card / side insert 会被误救回正文 +3. `page` 号和人名信号都不该作为主判定依据 + +目标是把这条线改成: + +- `authors` 只走 `frontmatter zone + Zotero/source metadata anchor` +- `author bio` / `profile card` / `early-page side insert` 统一走 `non_body_insert` 路径 +- 先识别正文主 spine,再识别“不属于正文”的插入 cluster +- 人名/履历句式只做增强信号,不做主判定 + +## Current Root Cause + +`SAN9AYVR` 暴露了三类错误: + +### 1. 正文被误判成 `authors` + +例如 page 3 的: + +- `In Section 5, the focus is on ES based bioelectronics ... $^{8,49}$` + +被打成 `authors`。 + +根因: + +- 当前 `ocr_roles.py` 对 author-like text 仍然开放式识别 +- superscript / comma / name-like token 在综述正文里很容易误触发 + +### 2. page-1 / page-2 的作者简介块被当正文 + +例如: + +- `Dr Ya Huang is currently ...` +- `Zhenlin Chen is currently ...` +- `Xinge Yu is the Associate Director ...` + +这些在 PDF 上是明显的 early-page 插入 profile blocks,不属于正文主链。 +但现在要么直接被判 `body_paragraph`,要么先成 `frontmatter_noise` 再被 rescue 成 `body_paragraph`。 + +根因: + +- 系统还缺少 `non_body_insert` 这一类结构类型 +- rescue 只看“像不像 body”,没先问“它是否属于 body spine” + +### 3. `authors` 和 `author bio` 的职责混在一起 + +现在 `authors` 被拿来承担两种事情: + +- metadata/frontmatter 作者块 +- 各种 name-like non-body blocks + +这是不对的。两者必须拆开: + +- `authors`:frontmatter metadata anchor +- `author_bio` / `profile insert`:non-body insert + +## Design Principles + +### 1. `authors` must be anchored, not guessed + +`authors` 只允许来自: + +- page-1 `frontmatter zone` +- 与 Zotero/source metadata authors list 有足够匹配的 OCR block + +禁止: + +- 从正文任意 block 开放式猜 `authors` +- 仅因有人名样式/上标/逗号就升成 `authors` + +### 2. Determine body spine first + +不是先识别 “sidebar”。 + +而是: + +1. 先识别正文主 spine +2. 再找不属于 body spine 的 early-page insert cluster +3. 最后再用人名/履历语义做弱确认 + +### 3. Name-like text is only a confidence booster + +人名信号只能在以下前提后使用: + +- 该 block 已被判定为 non-body cluster 成员 + +用途: + +- 增强 `profile_insert` / `author_bio_insert` 置信度 + +不能: + +- 直接产出 `authors` +- 直接产出 `sidebar` + +### 4. Avoid absolute page heuristics + +不要写: + +- `page <= 3 -> maybe sidebar` +- `page == 2 -> maybe author bio` + +可以使用的只有弱相对先验: + +- early document region +- close to frontmatter/body transition +- detached from body spine + +但 page 号不能单独决定角色。 + +## Target End State + +After this plan: + +- metadata authors come from Zotero/source metadata truth, optionally aligned to a real OCR block +- body text no longer gets promoted to `authors` +- early-page author biography/profile cards become `non_body_insert`-family blocks +- rescue no longer converts non-body insert blocks back into `body_paragraph` + +## Implementation Plan + +### Task 1: Remove open-ended `authors` text heuristics + +Files: + +- `paperforge/worker/ocr_roles.py` +- tests in `tests/test_ocr_roles.py` + +#### 1.1 Remove generic author promotion from `text` blocks + +Find and delete or strongly demote logic that promotes arbitrary text to `authors` based on: + +- superscript affiliation markers +- commas +- name-like shapes +- lack of year parens + +This logic is currently too broad and causes正文误识别. + +#### 1.2 Keep `authors` only in frontmatter zone + +Allow `authors` assignment only when: + +- page-1 zone detector returns `author_zone` +- or a future anchored author-candidate check explicitly confirms it + +All other blocks should fall through to: + +- `body_paragraph` +- `unknown_structural` +- or later `non_body_insert` + +Acceptance: + +- body blocks can no longer become `authors` purely from text shape + +### Task 2: Add Zotero-anchored author candidate resolution + +Files: + +- `paperforge/worker/ocr_metadata.py` +- optionally `ocr_roles.py` +- tests in `tests/test_ocr_metadata.py` + +#### 2.1 Treat source metadata authors as truth + +For resolved metadata: + +- if Zotero/source metadata authors exist, they are canonical +- OCR authors should only serve as alignment/traceability support + +#### 2.2 Add OCR block alignment, not OCR author guessing + +Implement a helper along the lines of: + +- `_match_author_block_to_source_authors(blocks, source_authors)` + +Use: + +- page-1 candidate blocks only +- normalized name matching +- token overlap / fuzzy similarity + +If matched: + +- keep block role as `authors` +- store raw OCR authors evidence if useful + +If not matched: + +- metadata still uses source authors +- do not invent OCR `authors` + +Acceptance: + +- wrong OCR `authors` block does not pollute `resolved_metadata.authors` + +### Task 3: Introduce `non_body_insert` / `profile_insert` regime + +Files: + +- `paperforge/worker/ocr_document.py` +- `paperforge/worker/ocr_blocks.py` +- `ocr_render.py` if render suppression is needed +- tests in `tests/test_ocr_document.py`, `tests/test_ocr_render_stabilization.py` + +#### 3.1 Add a document-level detector for early-page non-body insert clusters + +New helper(s), for example: + +- `_detect_body_spine(blocks)` +- `_detect_non_body_insert_clusters(blocks, body_spine, ...)` + +The detector should rely primarily on: + +- geometry +- block continuity +- cluster layout +- style coherence + +Not on fixed text phrases. + +#### 3.2 What counts as a candidate cluster + +Signals: + +- block is not on the main body spine +- block belongs to an early-page region near the body/frontmatter transition +- multiple similar blocks appear in a local grid / side cluster +- cluster width / line height / family differ from body spine +- there is a visible vertical separation from body paragraphs + +Optional weak boost: + +- name-like lead +- biography-like tense or CV-style sentence forms + +#### 3.3 Role strategy + +Do not overload `authors`. + +Either: + +- introduce explicit role `non_body_insert` + +or: + +- add `_non_body_insert = true` regime marker while keeping `body_paragraph` temporarily + +Recommendation: + +- use explicit role `non_body_insert` + +This keeps downstream logic simpler. + +Acceptance: + +- page 1/2 author bio blocks in `SAN9AYVR` no longer belong to body spine + +### Task 4: Block rescue from pulling insert clusters back into body + +Files: + +- `paperforge/worker/ocr_document.py` +- tests in `tests/test_ocr_document.py` + +#### 4.1 Add rescue guard + +In `rescue_roles_with_document_context()`: + +- before rescuing `frontmatter_noise -> body_paragraph` +- or similar body-promoting logic + +check: + +- whether the block belongs to a detected `non_body_insert` cluster + +If yes: + +- do not rescue to body + +#### 4.2 Body-like font is not enough + +This is the key fix for `SAN9AYVR`: + +- page-1/2 author bio blocks have body-like font size +- but they are still not body + +So rescue must require: + +- body family match +- and body-spine compatibility + +not just font match. + +Acceptance: + +- author bio/profile insert blocks stay out of正文 even when their font matches body + +### Task 5: Use style/geometry more explicitly for insert detection + +Files: + +- `paperforge/worker/ocr_profiles.py` +- `paperforge/worker/ocr_document.py` + +#### 5.1 Add body-spine comparison helpers + +Useful block-level features: + +- block width +- line height distribution +- mean font size +- font family +- italic/bold ratios +- x-column placement +- gap above/below + +Use these to compare candidate blocks to the learned body spine. + +#### 5.2 Detection logic should be relative, not absolute + +No: + +- fixed page number +- fixed width thresholds without page-relative normalization + +Yes: + +- narrower/wider than body family on the same paper +- detached from body chain on the same page/spread +- repeated cluster pattern on the same paper + +Acceptance: + +- insert detection generalizes across journals better than phrase rules + +### Task 6: Downstream behavior + +Files: + +- `ocr_metadata.py` +- `ocr_render.py` +- maybe `ocr_index.py` + +#### 6.1 Metadata + +- `non_body_insert` must not be used as author candidate +- only anchored `authors` blocks may feed OCR frontmatter candidates + +#### 6.2 Render + +Default: + +- `non_body_insert` should not render in `fulltext.md` + +Optional: + +- if needed later, preserve them in separate object/appendix notes +- but do not include in正文 + +#### 6.3 Index + +- keep them out of main body index buckets +- optionally add a separate bucket later if useful + +## Testing Plan + +### Unit / integration tests + +1. `tests/test_ocr_roles.py` +- body paragraph with superscript citations must not become `authors` +- page-1 author zone still yields `authors` + +2. `tests/test_ocr_metadata.py` +- source/Zotero authors override bad OCR author candidates +- OCR alignment works only for page-1 candidates + +3. `tests/test_ocr_document.py` +- early-page insert cluster is detected as non-body +- non-body insert is not rescued to `body_paragraph` + +4. `tests/test_ocr_render_stabilization.py` +- `non_body_insert` does not appear in rendered正文 + +### Real-paper validation + +Primary: + +- `SAN9AYVR` + +Must verify: + +- `resolved_metadata.authors` is correct +- author bio blocks on pages 1-2 are absent from正文 +- page 3 `In Section 5 ...` is body, not authors + +Secondary regression: + +- `2GN9LMCW` +- `7C8829BD` + +Must verify: + +- no regression in frontmatter authors +- no regression in tail/backmatter structure + +## Acceptance Criteria + +This plan is complete when: + +1. arbitrary正文 text can no longer be promoted to `authors` +2. metadata authors are anchored to source/Zotero truth +3. early-page author biographies/profile cards are classified as non-body insert blocks +4. rescue no longer drags these insert blocks back into正文 +5. `SAN9AYVR/fulltext.md` no longer contains author biography prose + +## Risks + +1. Overfitting insert detection to this one paper +- avoid “is currently / received degree” as primary rules + +2. Hiding legitimate boxed正文 content +- keep detection cluster-based and relative to body spine +- validate on at least one paper with real正文 side notes later + +3. Confusing affiliation/footnote blocks with author bios +- author bios are paragraph-like insert clusters +- affiliation blocks remain page-1 frontmatter candidates + diff --git a/docs/superpowers/plans/2026-06-06-ocr-structure-truth-convergence-plan.md b/docs/superpowers/plans/2026-06-06-ocr-structure-truth-convergence-plan.md new file mode 100644 index 00000000..147e60f3 --- /dev/null +++ b/docs/superpowers/plans/2026-06-06-ocr-structure-truth-convergence-plan.md @@ -0,0 +1,480 @@ +# OCR Structure Truth Convergence Plan + +Date: 2026-06-06 +Scope: `paperforge/worker/ocr_pdf_spans.py`, `ocr_blocks.py`, `ocr_document.py`, `ocr_roles.py`, `ocr_profiles.py`, `ocr_render.py`, `ocr_rebuild.py`, downstream tests + +## Goal + +收掉当前 structured OCR pipeline 里最危险的三条分叉,让全文结构、span 样式、最终 `fulltext.md` 使用同一套真相: + +1. `span_metadata` 必须基于正确坐标系提取 +2. `DocumentStructure` 必须成为唯一的全文结构真相 +3. `role/profile` 的构建顺序必须和结构归一化顺序一致 + +这份计划不重做 pipeline。它从当前代码状态出发,收敛现有实现。 + +## Current Root Causes + +### 1. Span extraction likely uses the wrong coordinate system + +Current behavior: + +- `ocr_pdf_spans.extract_pdf_spans_for_block()` directly uses OCR block bbox as `page.get_text("rawdict", clip=rect)` PDF coordinates. +- OCR block bbox is derived from rendered page image coordinates, not guaranteed PDF coordinates. +- We already hit the same coordinate mismatch in figure/table cropping earlier. + +Impact: + +- `span_metadata` may be extracted from the wrong PDF region +- style-based role inference becomes noisy or silently empty +- all later span-profile logic rests on an unstable base + +### 2. Document structure is analyzed twice, in two different layers + +Current behavior: + +- `ocr_blocks.build_structured_blocks()`: + - seeds roles + - calls `analyze_document_structure()` + - calls `rescue_roles_with_document_context()` +- `ocr_render._order_tail_blocks()`: + - calls `analyze_document_structure()` again + - does additional `_promote_tail_body_candidates()` + - does additional `_assign_tail_spread_ownership()` + +Impact: + +- structure layer and render layer can diverge +- `fulltext.md` may not reflect the same truth used by metadata/index/health/inventory +- debugging remains difficult because render still mutates structural semantics + +### 3. Profile construction happens before structural normalization settles + +Current behavior: + +- `ocr_blocks.build_structured_blocks()`: + - first-pass roles + - builds `role_profiles` + - only then runs `analyze_document_structure()` +- `analyze_document_structure()` currently normalizes some roles in-place: + - boundary/container backmatter normalization + - body/frontmatter noise to backmatter body in boundary zone + +Impact: + +- `role_span_profiles.json` represents pre-normalized roles +- rescue logic compares blocks against profiles that are partially stale +- backmatter families are especially under-modeled + +## Non-Goals + +- Do not redesign figure/table pipeline here +- Do not add new journal-specific text patches +- Do not introduce absolute page gates or absolute font-size thresholds as primary logic +- Do not rebuild the pipeline from scratch + +## Target End State + +The final pipeline should be: + +1. raw blocks +2. span extraction on correct coordinates +3. seed roles only +4. document structure analysis and structural normalization +5. profile building on normalized structure +6. section-aware rescue +7. downstream consumers render/index/health/inventory consume the same structure without re-inferring it + +In other words: + +- `ocr_roles.py` owns seed-role assignment only +- `ocr_document.py` owns document segmentation and structural normalization +- `ocr_profiles.py` owns profile construction and profile comparison +- `ocr_render.py` renders; it does not invent structure + +## Implementation Plan + +### Task 1: Fix span extraction coordinate baseline + +Files: + +- `paperforge/worker/ocr_pdf_spans.py` +- `paperforge/worker/ocr.py` +- `paperforge/worker/ocr_rebuild.py` +- tests in `tests/test_ocr_pdf_spans.py` + +#### 1.1 Add an explicit OCR-to-PDF coordinate mapping path + +Update `extract_pdf_spans_for_block()` and/or introduce a helper such as: + +- `_map_ocr_bbox_to_pdf_rect(page, bbox, page_width, page_height)` + +Use: + +- OCR page dimensions from `raw_blocks.page_width/page_height` +- PDF page rect dimensions from `fitz.Page.rect` + +Expected mapping: + +- if OCR bbox is in rendered image coordinates, convert it back into PDF coordinates using scale ratios +- avoid direct `fitz.Rect(*bbox)` on OCR image coordinates + +#### 1.2 Carry the required dimensions through the API + +Update call sites so span extraction receives: + +- `page_width` +- `page_height` + +Likely signature change: + +- `extract_pdf_spans_for_block(pdf_doc, page_num, bbox, page_width=None, page_height=None)` + +#### 1.3 Preserve graceful degradation + +If mapping inputs are absent or invalid: + +- return `None` +- do not crash rebuild/ocr + +#### 1.4 Add tests for coordinate mapping + +Add tests that verify: + +- OCR-space bbox is converted before clip extraction +- no-span fallback remains stable when PDF is missing or bbox invalid + +Acceptance: + +- `span_metadata` extraction no longer relies on raw OCR bbox as PDF clip coords +- tests cover mapped extraction path + +### Task 2: Make `ocr_roles.py` a pure seed-role pass + +Files: + +- `paperforge/worker/ocr_roles.py` +- tests in `tests/test_ocr_roles.py` + +#### 2.1 Remove residual structure-owning logic from role assignment + +`assign_block_role()` should keep: + +- raw label priors +- page-1 frontmatter zone heuristics +- figure/table formal prefix detection +- obvious abstract/reference/backmatter heading seeds +- conservative body fallback + +It should stop owning: + +- late tail spread ownership +- cross-page tail continuation logic +- render-like backmatter reordering assumptions + +#### 2.2 Demote high-risk hard rules + +Specifically remove or weaken as primary rules: + +- `_is_backmatter_boundary_heading()` relative page gate: + - `if total_pages > 0 and (page_num / total_pages) < 0.5: return False` +- visual heading shortcuts like: + - `font_size >= 14` + - `bold && font_size >= 11` + +Replace with: + +- heading-like style as one weak signal +- let `ocr_document.py` and later profile matching validate boundary families + +#### 2.3 Keep boundary seeds conservative + +`backmatter_boundary_heading` should remain seedable, but only when multiple signals agree: + +- heading-like shape +- candidate container wording or structure +- not a known backmatter child heading +- not references + +No absolute page number or hard half-document gate. + +Acceptance: + +- `assign_block_role()` becomes easier to reason about as a seed pass +- tests no longer assert page-ratio gating behavior + +### Task 3: Move all document-level structural mutation into `ocr_document.py` + +Files: + +- `paperforge/worker/ocr_document.py` +- `paperforge/worker/ocr_render.py` +- tests in `tests/test_ocr_document.py`, `tests/test_ocr_render_stabilization.py` + +#### 3.1 Define `ocr_document.py` as the only owner of structural normalization + +`ocr_document.py` should own: + +- body/backmatter/references boundary detection +- tail spread reconciliation +- backmatter form classification +- backmatter role normalization after boundary +- tail body candidate promotion +- tail spread ownership assignment + +Currently some of this logic still lives in `ocr_render.py`: + +- `_promote_tail_body_candidates()` +- `_assign_tail_spread_ownership()` + +Move these into `ocr_document.py` or make `ocr_render.py` call normalized results only. + +#### 3.2 Make `analyze_document_structure()` return normalized blocks explicitly + +Current issue: + +- it mutates blocks in-place indirectly and returns only `DocumentStructure` + +Refactor to something like: + +- `analyze_document_structure(blocks) -> tuple[DocumentStructure, list[dict]]` + +or: + +- `normalize_document_structure(blocks) -> NormalizedDocument` + +where the returned artifact includes: + +- `document_structure` +- `normalized_blocks` + +This avoids hidden in-place mutation and makes downstream ordering explicit. + +#### 3.3 Stop `ocr_render.py` from re-owning structure + +`render_fulltext_markdown()` should consume already-normalized blocks and a precomputed `DocumentStructure`. + +It should not: + +- call `analyze_document_structure()` again +- promote tail bodies +- assign tail ownership + +It may still do presentation-only ordering inside already-normalized page groups, but not semantic role mutation. + +Acceptance: + +- only one module owns structure mutation +- render consumes structure, it does not invent it + +### Task 4: Reorder profile construction after structural normalization + +Files: + +- `paperforge/worker/ocr_blocks.py` +- `paperforge/worker/ocr_profiles.py` +- `paperforge/worker/ocr_document.py` +- tests in `tests/test_ocr_document.py`, `tests/test_ocr_profiles.py` if present + +#### 4.1 Change `build_structured_blocks()` order + +Current order: + +1. seed roles +2. build role profiles +3. analyze document structure +4. rescue + +Target order: + +1. seed roles +2. analyze document structure + normalize roles +3. build profiles on normalized roles +4. rescue with normalized roles and normalized profiles + +This is the most important sequencing fix. + +#### 4.2 Expand profile semantics from role-only toward family-aware + +Do not throw away `build_role_span_profiles()`. +Extend it so later rescue can reason about: + +- body family +- heading family +- backmatter heading family +- reference family +- caption family + +Minimal first step: + +- keep role profiles for compatibility +- add helper(s) to derive family views from normalized roles + +Do not overdesign this phase; the main goal here is correct sequencing, not a full new taxonomy. + +Acceptance: + +- `role_span_profiles.json` is built from normalized structure, not pre-normalized seeds +- backmatter/reference/body families are less stale + +### Task 5: Upgrade rescue to use normalized structure as context + +Files: + +- `paperforge/worker/ocr_document.py` +- `paperforge/worker/ocr_blocks.py` +- tests in `tests/test_ocr_document.py` + +#### 5.1 Keep rescue narrow but make it structurally consistent + +Current rescue is acceptable in scope, but must operate on: + +- normalized blocks +- normalized profiles +- final section boundaries + +It should continue to own: + +- frontmatter noise rescued back to body +- body promoted to reference in references zone +- weak heading demoted to body + +#### 5.2 Prevent rescue from fighting stale roles + +Once Task 4 is done, rescue no longer compares against pre-boundary profiles. + +That should especially improve: + +- backmatter child stability +- reference consistency +- weak heading/body disambiguation + +Acceptance: + +- rescue uses normalized context only +- no rescue step depends on pre-normalization role families + +### Task 6: Make downstream products consume the same structure truth + +Files: + +- `paperforge/worker/ocr_rebuild.py` +- `paperforge/worker/ocr.py` +- `paperforge/worker/ocr_render.py` +- possibly `ocr_health.py`, `ocr_index.py` + +#### 6.1 Thread document-structure output through downstream calls + +Where possible, pass: + +- normalized structured blocks +- document structure +- normalized profiles + +into downstream stages instead of forcing each layer to reconstruct assumptions. + +Minimal requirement: + +- render must not recompute structure + +Optional but desirable: + +- health/index may later write structure summaries for debugging + +#### 6.2 Consider persisting document structure artifact + +Add a derived artifact such as: + +- `structure/document_structure.json` + +Contents may include: + +- `body_end_page` +- `backmatter_start` +- `references_start` +- `spread_start` +- `spread_end` +- `backmatter_form` + +This is useful for: + +- debugging +- reproducibility +- making render/index/health easier to compare + +Acceptance: + +- same structural truth is visible and reusable downstream + +## Testing Plan + +### Unit / integration targets + +1. `tests/test_ocr_pdf_spans.py` +- mapped OCR-to-PDF coordinate extraction +- graceful fallback + +2. `tests/test_ocr_roles.py` +- remove expectations that depend on hard page gates or hard font thresholds +- keep seed-role expectations conservative + +3. `tests/test_ocr_document.py` +- boundary detection still works +- backmatter normalization still works +- rescue uses normalized roles + +4. `tests/test_ocr_render_stabilization.py` +- render no longer depends on its own structural mutation +- tail ordering remains correct when given normalized blocks + +5. if needed, add a new test around `build_structured_blocks()` ordering +- asserts profile build happens after document normalization + +### Real-paper regression set + +Must re-run on: + +- `7C8829BD` + - Frontiers-style mixed tail spread +- `2GN9LMCW` + - PeerJ-style declarations container + references + unnumbered headings + +Verification focus: + +- structure layer and rendered markdown agree on backmatter order +- references remain stable +- no reintroduced page marker mismatch +- span metadata is present where expected and not obviously empty/misaligned + +## Acceptance Criteria + +This plan is complete when all of the following are true: + +1. `span_metadata` extraction no longer assumes OCR bbox == PDF coordinates +2. `ocr_render.py` does not recompute or mutate document structure semantics +3. `build_structured_blocks()` builds profiles after structural normalization +4. rescue operates on normalized roles and normalized profiles +5. `7C8829BD` and `2GN9LMCW` remain correct after rebuild +6. tests cover the new sequencing and coordinate-baseline assumptions + +## Risks To Watch + +1. Over-coupling render to a new structure artifact too early +- keep the first change minimal: pass normalized blocks first, persist structure later if useful + +2. Breaking old tests that implicitly relied on hard threshold behavior +- update tests toward seed-role behavior, not old heuristic internals + +3. Span extraction still returning sparse data on scan-heavy PDFs +- that is acceptable if graceful degradation remains intact +- the important fix is to stop reading obviously wrong regions + +## Recommended Execution Order + +1. Task 1 — span coordinate fix +2. Task 3 — make `ocr_document.py` the structure owner +3. Task 4 — reorder profile construction +4. Task 5 — rescue on normalized context +5. Task 6 — downstream truth convergence +6. finalize tests and real-paper rebuild verification + diff --git a/paperforge/worker/ocr_roles.py b/paperforge/worker/ocr_roles.py index d93ebf90..47d50ca3 100644 --- a/paperforge/worker/ocr_roles.py +++ b/paperforge/worker/ocr_roles.py @@ -591,26 +591,6 @@ def assign_block_role( evidence=[f"citation line pattern: {text[:60]}"], ) - # Author list with superscript affiliation markers → authors, not noise - # Distinguish from citation lines (which have year in parens) and - # affiliation blocks (which have institutional keywords) - has_year_parens = bool(re.search(r"\(\d{4}[a-z]?\)", text)) - has_inst_keyword = any( - kw in lower_txt for kw in ["department", "university", "institute", "college", "school of"] - ) - if ( - _AUTHOR_AFFILIATION_MARKER.search(text) - and "," in text - and not has_year_parens - and not has_inst_keyword - and len(text) < 500 - ): - return RoleAssignment( - role="authors", - confidence=0.8, - evidence=[f"author list with affiliation markers: {text[:60]}"], - ) - # Affiliation block starting with superscript (page 1 only) if ( still_frontmatter diff --git a/tests/test_ocr_roles.py b/tests/test_ocr_roles.py index f79d50d5..3627a61f 100644 --- a/tests/test_ocr_roles.py +++ b/tests/test_ocr_roles.py @@ -343,6 +343,31 @@ def test_heading_level_from_profile_match() -> None: pass # Placeholder — will be updated after refactor +def test_body_citation_not_authors() -> None: + from paperforge.worker.ocr_roles import assign_block_role + block = { + "block_label": "text", + "block_content": "In Section 5, the focus is on ES based bioelectronics $^{8,49}$", + "block_bbox": [100, 500, 800, 540], + "page": 3, + } + result = assign_block_role(block, page_blocks=[block], page_width=1200, page_height=1600) + assert result.role == "body_paragraph", f"Expected body_paragraph, got {result.role}" + assert result.confidence >= 0.5 + + +def test_frontmatter_author_zone_still_works() -> None: + from paperforge.worker.ocr_roles import assign_block_role + block = { + "block_label": "text", + "block_content": "Alice Smith, Bob Jones, Charlie Brown", + "block_bbox": [100, 300, 800, 330], + "page": 1, + } + result = assign_block_role(block, page_blocks=[block], page_width=1200, page_height=1600) + assert result.role == "authors", f"Expected authors, got {result.role}" + + def test_backmatter_boundary_detects_on_early_page() -> None: """Backmatter boundary should be detectable on papers with fewer than 8 pages, without a hard page gate."""