mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: remove open-ended authors text heuristics, anchor to page-1 zone only
This commit is contained in:
Research Assistant
parent
2bf7e73fff
commit
c974ba7480
5 changed files with 994 additions and 20 deletions
50
_check_rebuild.py
Normal file
50
_check_rebuild.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
from pathlib import Path
|
||||
import json
|
||||
|
||||
vault = Path(r"D:\L\OB\Literature-hub")
|
||||
structured_path = vault / "System" / "PaperForge" / "ocr" / "SAN9AYVR" / "blocks.structured.jsonl"
|
||||
profiles_path = vault / "System" / "PaperForge" / "ocr" / "SAN9AYVR" / "role_span_profiles.json"
|
||||
|
||||
if structured_path.exists():
|
||||
print(f"structured blocks: {structured_path.stat().st_size} bytes")
|
||||
with open(structured_path, encoding="utf-8") as f:
|
||||
line = f.readline()
|
||||
row = json.loads(line)
|
||||
has_span = "span_metadata" in row
|
||||
has_role = row.get("role")
|
||||
has_conf = row.get("role_confidence")
|
||||
print(f" first block role={has_role}, conf={has_conf}, has_span_metadata={has_span}")
|
||||
|
||||
total = 0
|
||||
with_span = 0
|
||||
with open(structured_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
total += 1
|
||||
row = json.loads(line)
|
||||
if row.get("span_metadata"):
|
||||
with_span += 1
|
||||
print(f" total blocks: {total}, with span_metadata: {with_span} ({100*with_span/total:.1f}%)")
|
||||
else:
|
||||
print("structured blocks not found")
|
||||
|
||||
if profiles_path.exists():
|
||||
profiles = json.loads(profiles_path.read_text(encoding="utf-8"))
|
||||
print(f"\nrole_span_profiles.json: {len(profiles)} roles")
|
||||
for role, p in sorted(profiles.items()):
|
||||
print(f" {role}: count={p['block_count']}, size={p['mean_size']}, quality={p['quality']}")
|
||||
else:
|
||||
print("role_span_profiles.json not found")
|
||||
|
||||
second_pass_total = 0
|
||||
span_alt_total = 0
|
||||
with open(structured_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
row = json.loads(line)
|
||||
ev = row.get("evidence", [])
|
||||
for e in ev:
|
||||
if "second_pass" in e:
|
||||
second_pass_total += 1
|
||||
if "span_alternatives" in e:
|
||||
span_alt_total += 1
|
||||
print(f"\nSecond-pass applied: {second_pass_total} blocks")
|
||||
print(f"Span alternatives flagged: {span_alt_total} blocks")
|
||||
|
|
@ -0,0 +1,439 @@
|
|||
# OCR Author Anchor And Non-Body Insert Plan
|
||||
|
||||
Date: 2026-06-06
|
||||
Scope: `ocr_roles.py`, `ocr_document.py`, `ocr_metadata.py`, `ocr_blocks.py`, related tests, real-paper validation on `SAN9AYVR`
|
||||
|
||||
## Goal
|
||||
|
||||
收掉当前最不稳的一条链：
|
||||
|
||||
1. 开放式 `authors` 文本识别会误伤正文
|
||||
2. early-page 作者简介 / profile card / side insert 会被误救回正文
|
||||
3. `page` 号和人名信号都不该作为主判定依据
|
||||
|
||||
目标是把这条线改成：
|
||||
|
||||
- `authors` 只走 `frontmatter zone + Zotero/source metadata anchor`
|
||||
- `author bio` / `profile card` / `early-page side insert` 统一走 `non_body_insert` 路径
|
||||
- 先识别正文主 spine，再识别“不属于正文”的插入 cluster
|
||||
- 人名/履历句式只做增强信号，不做主判定
|
||||
|
||||
## Current Root Cause
|
||||
|
||||
`SAN9AYVR` 暴露了三类错误：
|
||||
|
||||
### 1. 正文被误判成 `authors`
|
||||
|
||||
例如 page 3 的：
|
||||
|
||||
- `In Section 5, the focus is on ES based bioelectronics ... $^{8,49}$`
|
||||
|
||||
被打成 `authors`。
|
||||
|
||||
根因：
|
||||
|
||||
- 当前 `ocr_roles.py` 对 author-like text 仍然开放式识别
|
||||
- superscript / comma / name-like token 在综述正文里很容易误触发
|
||||
|
||||
### 2. page-1 / page-2 的作者简介块被当正文
|
||||
|
||||
例如：
|
||||
|
||||
- `Dr Ya Huang is currently ...`
|
||||
- `Zhenlin Chen is currently ...`
|
||||
- `Xinge Yu is the Associate Director ...`
|
||||
|
||||
这些在 PDF 上是明显的 early-page 插入 profile blocks，不属于正文主链。
|
||||
但现在要么直接被判 `body_paragraph`，要么先成 `frontmatter_noise` 再被 rescue 成 `body_paragraph`。
|
||||
|
||||
根因：
|
||||
|
||||
- 系统还缺少 `non_body_insert` 这一类结构类型
|
||||
- rescue 只看“像不像 body”，没先问“它是否属于 body spine”
|
||||
|
||||
### 3. `authors` 和 `author bio` 的职责混在一起
|
||||
|
||||
现在 `authors` 被拿来承担两种事情：
|
||||
|
||||
- metadata/frontmatter 作者块
|
||||
- 各种 name-like non-body blocks
|
||||
|
||||
这是不对的。两者必须拆开：
|
||||
|
||||
- `authors`：frontmatter metadata anchor
|
||||
- `author_bio` / `profile insert`：non-body insert
|
||||
|
||||
## Design Principles
|
||||
|
||||
### 1. `authors` must be anchored, not guessed
|
||||
|
||||
`authors` 只允许来自：
|
||||
|
||||
- page-1 `frontmatter zone`
|
||||
- 与 Zotero/source metadata authors list 有足够匹配的 OCR block
|
||||
|
||||
禁止：
|
||||
|
||||
- 从正文任意 block 开放式猜 `authors`
|
||||
- 仅因有人名样式/上标/逗号就升成 `authors`
|
||||
|
||||
### 2. Determine body spine first
|
||||
|
||||
不是先识别 “sidebar”。
|
||||
|
||||
而是：
|
||||
|
||||
1. 先识别正文主 spine
|
||||
2. 再找不属于 body spine 的 early-page insert cluster
|
||||
3. 最后再用人名/履历语义做弱确认
|
||||
|
||||
### 3. Name-like text is only a confidence booster
|
||||
|
||||
人名信号只能在以下前提后使用：
|
||||
|
||||
- 该 block 已被判定为 non-body cluster 成员
|
||||
|
||||
用途：
|
||||
|
||||
- 增强 `profile_insert` / `author_bio_insert` 置信度
|
||||
|
||||
不能：
|
||||
|
||||
- 直接产出 `authors`
|
||||
- 直接产出 `sidebar`
|
||||
|
||||
### 4. Avoid absolute page heuristics
|
||||
|
||||
不要写：
|
||||
|
||||
- `page <= 3 -> maybe sidebar`
|
||||
- `page == 2 -> maybe author bio`
|
||||
|
||||
可以使用的只有弱相对先验：
|
||||
|
||||
- early document region
|
||||
- close to frontmatter/body transition
|
||||
- detached from body spine
|
||||
|
||||
但 page 号不能单独决定角色。
|
||||
|
||||
## Target End State
|
||||
|
||||
After this plan:
|
||||
|
||||
- metadata authors come from Zotero/source metadata truth, optionally aligned to a real OCR block
|
||||
- body text no longer gets promoted to `authors`
|
||||
- early-page author biography/profile cards become `non_body_insert`-family blocks
|
||||
- rescue no longer converts non-body insert blocks back into `body_paragraph`
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Task 1: Remove open-ended `authors` text heuristics
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_roles.py`
|
||||
- tests in `tests/test_ocr_roles.py`
|
||||
|
||||
#### 1.1 Remove generic author promotion from `text` blocks
|
||||
|
||||
Find and delete or strongly demote logic that promotes arbitrary text to `authors` based on:
|
||||
|
||||
- superscript affiliation markers
|
||||
- commas
|
||||
- name-like shapes
|
||||
- lack of year parens
|
||||
|
||||
This logic is currently too broad and causes正文误识别.
|
||||
|
||||
#### 1.2 Keep `authors` only in frontmatter zone
|
||||
|
||||
Allow `authors` assignment only when:
|
||||
|
||||
- page-1 zone detector returns `author_zone`
|
||||
- or a future anchored author-candidate check explicitly confirms it
|
||||
|
||||
All other blocks should fall through to:
|
||||
|
||||
- `body_paragraph`
|
||||
- `unknown_structural`
|
||||
- or later `non_body_insert`
|
||||
|
||||
Acceptance:
|
||||
|
||||
- body blocks can no longer become `authors` purely from text shape
|
||||
|
||||
### Task 2: Add Zotero-anchored author candidate resolution
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_metadata.py`
|
||||
- optionally `ocr_roles.py`
|
||||
- tests in `tests/test_ocr_metadata.py`
|
||||
|
||||
#### 2.1 Treat source metadata authors as truth
|
||||
|
||||
For resolved metadata:
|
||||
|
||||
- if Zotero/source metadata authors exist, they are canonical
|
||||
- OCR authors should only serve as alignment/traceability support
|
||||
|
||||
#### 2.2 Add OCR block alignment, not OCR author guessing
|
||||
|
||||
Implement a helper along the lines of:
|
||||
|
||||
- `_match_author_block_to_source_authors(blocks, source_authors)`
|
||||
|
||||
Use:
|
||||
|
||||
- page-1 candidate blocks only
|
||||
- normalized name matching
|
||||
- token overlap / fuzzy similarity
|
||||
|
||||
If matched:
|
||||
|
||||
- keep block role as `authors`
|
||||
- store raw OCR authors evidence if useful
|
||||
|
||||
If not matched:
|
||||
|
||||
- metadata still uses source authors
|
||||
- do not invent OCR `authors`
|
||||
|
||||
Acceptance:
|
||||
|
||||
- wrong OCR `authors` block does not pollute `resolved_metadata.authors`
|
||||
|
||||
### Task 3: Introduce `non_body_insert` / `profile_insert` regime
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_document.py`
|
||||
- `paperforge/worker/ocr_blocks.py`
|
||||
- `ocr_render.py` if render suppression is needed
|
||||
- tests in `tests/test_ocr_document.py`, `tests/test_ocr_render_stabilization.py`
|
||||
|
||||
#### 3.1 Add a document-level detector for early-page non-body insert clusters
|
||||
|
||||
New helper(s), for example:
|
||||
|
||||
- `_detect_body_spine(blocks)`
|
||||
- `_detect_non_body_insert_clusters(blocks, body_spine, ...)`
|
||||
|
||||
The detector should rely primarily on:
|
||||
|
||||
- geometry
|
||||
- block continuity
|
||||
- cluster layout
|
||||
- style coherence
|
||||
|
||||
Not on fixed text phrases.
|
||||
|
||||
#### 3.2 What counts as a candidate cluster
|
||||
|
||||
Signals:
|
||||
|
||||
- block is not on the main body spine
|
||||
- block belongs to an early-page region near the body/frontmatter transition
|
||||
- multiple similar blocks appear in a local grid / side cluster
|
||||
- cluster width / line height / family differ from body spine
|
||||
- there is a visible vertical separation from body paragraphs
|
||||
|
||||
Optional weak boost:
|
||||
|
||||
- name-like lead
|
||||
- biography-like tense or CV-style sentence forms
|
||||
|
||||
#### 3.3 Role strategy
|
||||
|
||||
Do not overload `authors`.
|
||||
|
||||
Either:
|
||||
|
||||
- introduce explicit role `non_body_insert`
|
||||
|
||||
or:
|
||||
|
||||
- add `_non_body_insert = true` regime marker while keeping `body_paragraph` temporarily
|
||||
|
||||
Recommendation:
|
||||
|
||||
- use explicit role `non_body_insert`
|
||||
|
||||
This keeps downstream logic simpler.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- page 1/2 author bio blocks in `SAN9AYVR` no longer belong to body spine
|
||||
|
||||
### Task 4: Block rescue from pulling insert clusters back into body
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_document.py`
|
||||
- tests in `tests/test_ocr_document.py`
|
||||
|
||||
#### 4.1 Add rescue guard
|
||||
|
||||
In `rescue_roles_with_document_context()`:
|
||||
|
||||
- before rescuing `frontmatter_noise -> body_paragraph`
|
||||
- or similar body-promoting logic
|
||||
|
||||
check:
|
||||
|
||||
- whether the block belongs to a detected `non_body_insert` cluster
|
||||
|
||||
If yes:
|
||||
|
||||
- do not rescue to body
|
||||
|
||||
#### 4.2 Body-like font is not enough
|
||||
|
||||
This is the key fix for `SAN9AYVR`:
|
||||
|
||||
- page-1/2 author bio blocks have body-like font size
|
||||
- but they are still not body
|
||||
|
||||
So rescue must require:
|
||||
|
||||
- body family match
|
||||
- and body-spine compatibility
|
||||
|
||||
not just font match.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- author bio/profile insert blocks stay out of正文 even when their font matches body
|
||||
|
||||
### Task 5: Use style/geometry more explicitly for insert detection
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_profiles.py`
|
||||
- `paperforge/worker/ocr_document.py`
|
||||
|
||||
#### 5.1 Add body-spine comparison helpers
|
||||
|
||||
Useful block-level features:
|
||||
|
||||
- block width
|
||||
- line height distribution
|
||||
- mean font size
|
||||
- font family
|
||||
- italic/bold ratios
|
||||
- x-column placement
|
||||
- gap above/below
|
||||
|
||||
Use these to compare candidate blocks to the learned body spine.
|
||||
|
||||
#### 5.2 Detection logic should be relative, not absolute
|
||||
|
||||
No:
|
||||
|
||||
- fixed page number
|
||||
- fixed width thresholds without page-relative normalization
|
||||
|
||||
Yes:
|
||||
|
||||
- narrower/wider than body family on the same paper
|
||||
- detached from body chain on the same page/spread
|
||||
- repeated cluster pattern on the same paper
|
||||
|
||||
Acceptance:
|
||||
|
||||
- insert detection generalizes across journals better than phrase rules
|
||||
|
||||
### Task 6: Downstream behavior
|
||||
|
||||
Files:
|
||||
|
||||
- `ocr_metadata.py`
|
||||
- `ocr_render.py`
|
||||
- maybe `ocr_index.py`
|
||||
|
||||
#### 6.1 Metadata
|
||||
|
||||
- `non_body_insert` must not be used as author candidate
|
||||
- only anchored `authors` blocks may feed OCR frontmatter candidates
|
||||
|
||||
#### 6.2 Render
|
||||
|
||||
Default:
|
||||
|
||||
- `non_body_insert` should not render in `fulltext.md`
|
||||
|
||||
Optional:
|
||||
|
||||
- if needed later, preserve them in separate object/appendix notes
|
||||
- but do not include in正文
|
||||
|
||||
#### 6.3 Index
|
||||
|
||||
- keep them out of main body index buckets
|
||||
- optionally add a separate bucket later if useful
|
||||
|
||||
## Testing Plan
|
||||
|
||||
### Unit / integration tests
|
||||
|
||||
1. `tests/test_ocr_roles.py`
|
||||
- body paragraph with superscript citations must not become `authors`
|
||||
- page-1 author zone still yields `authors`
|
||||
|
||||
2. `tests/test_ocr_metadata.py`
|
||||
- source/Zotero authors override bad OCR author candidates
|
||||
- OCR alignment works only for page-1 candidates
|
||||
|
||||
3. `tests/test_ocr_document.py`
|
||||
- early-page insert cluster is detected as non-body
|
||||
- non-body insert is not rescued to `body_paragraph`
|
||||
|
||||
4. `tests/test_ocr_render_stabilization.py`
|
||||
- `non_body_insert` does not appear in rendered正文
|
||||
|
||||
### Real-paper validation
|
||||
|
||||
Primary:
|
||||
|
||||
- `SAN9AYVR`
|
||||
|
||||
Must verify:
|
||||
|
||||
- `resolved_metadata.authors` is correct
|
||||
- author bio blocks on pages 1-2 are absent from正文
|
||||
- page 3 `In Section 5 ...` is body, not authors
|
||||
|
||||
Secondary regression:
|
||||
|
||||
- `2GN9LMCW`
|
||||
- `7C8829BD`
|
||||
|
||||
Must verify:
|
||||
|
||||
- no regression in frontmatter authors
|
||||
- no regression in tail/backmatter structure
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
This plan is complete when:
|
||||
|
||||
1. arbitrary正文 text can no longer be promoted to `authors`
|
||||
2. metadata authors are anchored to source/Zotero truth
|
||||
3. early-page author biographies/profile cards are classified as non-body insert blocks
|
||||
4. rescue no longer drags these insert blocks back into正文
|
||||
5. `SAN9AYVR/fulltext.md` no longer contains author biography prose
|
||||
|
||||
## Risks
|
||||
|
||||
1. Overfitting insert detection to this one paper
|
||||
- avoid “is currently / received degree” as primary rules
|
||||
|
||||
2. Hiding legitimate boxed正文 content
|
||||
- keep detection cluster-based and relative to body spine
|
||||
- validate on at least one paper with real正文 side notes later
|
||||
|
||||
3. Confusing affiliation/footnote blocks with author bios
|
||||
- author bios are paragraph-like insert clusters
|
||||
- affiliation blocks remain page-1 frontmatter candidates
|
||||
|
||||
|
|
@ -0,0 +1,480 @@
|
|||
# OCR Structure Truth Convergence Plan
|
||||
|
||||
Date: 2026-06-06
|
||||
Scope: `paperforge/worker/ocr_pdf_spans.py`, `ocr_blocks.py`, `ocr_document.py`, `ocr_roles.py`, `ocr_profiles.py`, `ocr_render.py`, `ocr_rebuild.py`, downstream tests
|
||||
|
||||
## Goal
|
||||
|
||||
收掉当前 structured OCR pipeline 里最危险的三条分叉，让全文结构、span 样式、最终 `fulltext.md` 使用同一套真相：
|
||||
|
||||
1. `span_metadata` 必须基于正确坐标系提取
|
||||
2. `DocumentStructure` 必须成为唯一的全文结构真相
|
||||
3. `role/profile` 的构建顺序必须和结构归一化顺序一致
|
||||
|
||||
这份计划不重做 pipeline。它从当前代码状态出发，收敛现有实现。
|
||||
|
||||
## Current Root Causes
|
||||
|
||||
### 1. Span extraction likely uses the wrong coordinate system
|
||||
|
||||
Current behavior:
|
||||
|
||||
- `ocr_pdf_spans.extract_pdf_spans_for_block()` directly uses OCR block bbox as `page.get_text("rawdict", clip=rect)` PDF coordinates.
|
||||
- OCR block bbox is derived from rendered page image coordinates, not guaranteed PDF coordinates.
|
||||
- We already hit the same coordinate mismatch in figure/table cropping earlier.
|
||||
|
||||
Impact:
|
||||
|
||||
- `span_metadata` may be extracted from the wrong PDF region
|
||||
- style-based role inference becomes noisy or silently empty
|
||||
- all later span-profile logic rests on an unstable base
|
||||
|
||||
### 2. Document structure is analyzed twice, in two different layers
|
||||
|
||||
Current behavior:
|
||||
|
||||
- `ocr_blocks.build_structured_blocks()`:
|
||||
- seeds roles
|
||||
- calls `analyze_document_structure()`
|
||||
- calls `rescue_roles_with_document_context()`
|
||||
- `ocr_render._order_tail_blocks()`:
|
||||
- calls `analyze_document_structure()` again
|
||||
- does additional `_promote_tail_body_candidates()`
|
||||
- does additional `_assign_tail_spread_ownership()`
|
||||
|
||||
Impact:
|
||||
|
||||
- structure layer and render layer can diverge
|
||||
- `fulltext.md` may not reflect the same truth used by metadata/index/health/inventory
|
||||
- debugging remains difficult because render still mutates structural semantics
|
||||
|
||||
### 3. Profile construction happens before structural normalization settles
|
||||
|
||||
Current behavior:
|
||||
|
||||
- `ocr_blocks.build_structured_blocks()`:
|
||||
- first-pass roles
|
||||
- builds `role_profiles`
|
||||
- only then runs `analyze_document_structure()`
|
||||
- `analyze_document_structure()` currently normalizes some roles in-place:
|
||||
- boundary/container backmatter normalization
|
||||
- body/frontmatter noise to backmatter body in boundary zone
|
||||
|
||||
Impact:
|
||||
|
||||
- `role_span_profiles.json` represents pre-normalized roles
|
||||
- rescue logic compares blocks against profiles that are partially stale
|
||||
- backmatter families are especially under-modeled
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Do not redesign figure/table pipeline here
|
||||
- Do not add new journal-specific text patches
|
||||
- Do not introduce absolute page gates or absolute font-size thresholds as primary logic
|
||||
- Do not rebuild the pipeline from scratch
|
||||
|
||||
## Target End State
|
||||
|
||||
The final pipeline should be:
|
||||
|
||||
1. raw blocks
|
||||
2. span extraction on correct coordinates
|
||||
3. seed roles only
|
||||
4. document structure analysis and structural normalization
|
||||
5. profile building on normalized structure
|
||||
6. section-aware rescue
|
||||
7. downstream consumers render/index/health/inventory consume the same structure without re-inferring it
|
||||
|
||||
In other words:
|
||||
|
||||
- `ocr_roles.py` owns seed-role assignment only
|
||||
- `ocr_document.py` owns document segmentation and structural normalization
|
||||
- `ocr_profiles.py` owns profile construction and profile comparison
|
||||
- `ocr_render.py` renders; it does not invent structure
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Task 1: Fix span extraction coordinate baseline
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_pdf_spans.py`
|
||||
- `paperforge/worker/ocr.py`
|
||||
- `paperforge/worker/ocr_rebuild.py`
|
||||
- tests in `tests/test_ocr_pdf_spans.py`
|
||||
|
||||
#### 1.1 Add an explicit OCR-to-PDF coordinate mapping path
|
||||
|
||||
Update `extract_pdf_spans_for_block()` and/or introduce a helper such as:
|
||||
|
||||
- `_map_ocr_bbox_to_pdf_rect(page, bbox, page_width, page_height)`
|
||||
|
||||
Use:
|
||||
|
||||
- OCR page dimensions from `raw_blocks.page_width/page_height`
|
||||
- PDF page rect dimensions from `fitz.Page.rect`
|
||||
|
||||
Expected mapping:
|
||||
|
||||
- if OCR bbox is in rendered image coordinates, convert it back into PDF coordinates using scale ratios
|
||||
- avoid direct `fitz.Rect(*bbox)` on OCR image coordinates
|
||||
|
||||
#### 1.2 Carry the required dimensions through the API
|
||||
|
||||
Update call sites so span extraction receives:
|
||||
|
||||
- `page_width`
|
||||
- `page_height`
|
||||
|
||||
Likely signature change:
|
||||
|
||||
- `extract_pdf_spans_for_block(pdf_doc, page_num, bbox, page_width=None, page_height=None)`
|
||||
|
||||
#### 1.3 Preserve graceful degradation
|
||||
|
||||
If mapping inputs are absent or invalid:
|
||||
|
||||
- return `None`
|
||||
- do not crash rebuild/ocr
|
||||
|
||||
#### 1.4 Add tests for coordinate mapping
|
||||
|
||||
Add tests that verify:
|
||||
|
||||
- OCR-space bbox is converted before clip extraction
|
||||
- no-span fallback remains stable when PDF is missing or bbox invalid
|
||||
|
||||
Acceptance:
|
||||
|
||||
- `span_metadata` extraction no longer relies on raw OCR bbox as PDF clip coords
|
||||
- tests cover mapped extraction path
|
||||
|
||||
### Task 2: Make `ocr_roles.py` a pure seed-role pass
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_roles.py`
|
||||
- tests in `tests/test_ocr_roles.py`
|
||||
|
||||
#### 2.1 Remove residual structure-owning logic from role assignment
|
||||
|
||||
`assign_block_role()` should keep:
|
||||
|
||||
- raw label priors
|
||||
- page-1 frontmatter zone heuristics
|
||||
- figure/table formal prefix detection
|
||||
- obvious abstract/reference/backmatter heading seeds
|
||||
- conservative body fallback
|
||||
|
||||
It should stop owning:
|
||||
|
||||
- late tail spread ownership
|
||||
- cross-page tail continuation logic
|
||||
- render-like backmatter reordering assumptions
|
||||
|
||||
#### 2.2 Demote high-risk hard rules
|
||||
|
||||
Specifically remove or weaken as primary rules:
|
||||
|
||||
- `_is_backmatter_boundary_heading()` relative page gate:
|
||||
- `if total_pages > 0 and (page_num / total_pages) < 0.5: return False`
|
||||
- visual heading shortcuts like:
|
||||
- `font_size >= 14`
|
||||
- `bold && font_size >= 11`
|
||||
|
||||
Replace with:
|
||||
|
||||
- heading-like style as one weak signal
|
||||
- let `ocr_document.py` and later profile matching validate boundary families
|
||||
|
||||
#### 2.3 Keep boundary seeds conservative
|
||||
|
||||
`backmatter_boundary_heading` should remain seedable, but only when multiple signals agree:
|
||||
|
||||
- heading-like shape
|
||||
- candidate container wording or structure
|
||||
- not a known backmatter child heading
|
||||
- not references
|
||||
|
||||
No absolute page number or hard half-document gate.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- `assign_block_role()` becomes easier to reason about as a seed pass
|
||||
- tests no longer assert page-ratio gating behavior
|
||||
|
||||
### Task 3: Move all document-level structural mutation into `ocr_document.py`
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_document.py`
|
||||
- `paperforge/worker/ocr_render.py`
|
||||
- tests in `tests/test_ocr_document.py`, `tests/test_ocr_render_stabilization.py`
|
||||
|
||||
#### 3.1 Define `ocr_document.py` as the only owner of structural normalization
|
||||
|
||||
`ocr_document.py` should own:
|
||||
|
||||
- body/backmatter/references boundary detection
|
||||
- tail spread reconciliation
|
||||
- backmatter form classification
|
||||
- backmatter role normalization after boundary
|
||||
- tail body candidate promotion
|
||||
- tail spread ownership assignment
|
||||
|
||||
Currently some of this logic still lives in `ocr_render.py`:
|
||||
|
||||
- `_promote_tail_body_candidates()`
|
||||
- `_assign_tail_spread_ownership()`
|
||||
|
||||
Move these into `ocr_document.py` or make `ocr_render.py` call normalized results only.
|
||||
|
||||
#### 3.2 Make `analyze_document_structure()` return normalized blocks explicitly
|
||||
|
||||
Current issue:
|
||||
|
||||
- it mutates blocks in-place indirectly and returns only `DocumentStructure`
|
||||
|
||||
Refactor to something like:
|
||||
|
||||
- `analyze_document_structure(blocks) -> tuple[DocumentStructure, list[dict]]`
|
||||
|
||||
or:
|
||||
|
||||
- `normalize_document_structure(blocks) -> NormalizedDocument`
|
||||
|
||||
where the returned artifact includes:
|
||||
|
||||
- `document_structure`
|
||||
- `normalized_blocks`
|
||||
|
||||
This avoids hidden in-place mutation and makes downstream ordering explicit.
|
||||
|
||||
#### 3.3 Stop `ocr_render.py` from re-owning structure
|
||||
|
||||
`render_fulltext_markdown()` should consume already-normalized blocks and a precomputed `DocumentStructure`.
|
||||
|
||||
It should not:
|
||||
|
||||
- call `analyze_document_structure()` again
|
||||
- promote tail bodies
|
||||
- assign tail ownership
|
||||
|
||||
It may still do presentation-only ordering inside already-normalized page groups, but not semantic role mutation.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- only one module owns structure mutation
|
||||
- render consumes structure, it does not invent it
|
||||
|
||||
### Task 4: Reorder profile construction after structural normalization
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_blocks.py`
|
||||
- `paperforge/worker/ocr_profiles.py`
|
||||
- `paperforge/worker/ocr_document.py`
|
||||
- tests in `tests/test_ocr_document.py`, `tests/test_ocr_profiles.py` if present
|
||||
|
||||
#### 4.1 Change `build_structured_blocks()` order
|
||||
|
||||
Current order:
|
||||
|
||||
1. seed roles
|
||||
2. build role profiles
|
||||
3. analyze document structure
|
||||
4. rescue
|
||||
|
||||
Target order:
|
||||
|
||||
1. seed roles
|
||||
2. analyze document structure + normalize roles
|
||||
3. build profiles on normalized roles
|
||||
4. rescue with normalized roles and normalized profiles
|
||||
|
||||
This is the most important sequencing fix.
|
||||
|
||||
#### 4.2 Expand profile semantics from role-only toward family-aware
|
||||
|
||||
Do not throw away `build_role_span_profiles()`.
|
||||
Extend it so later rescue can reason about:
|
||||
|
||||
- body family
|
||||
- heading family
|
||||
- backmatter heading family
|
||||
- reference family
|
||||
- caption family
|
||||
|
||||
Minimal first step:
|
||||
|
||||
- keep role profiles for compatibility
|
||||
- add helper(s) to derive family views from normalized roles
|
||||
|
||||
Do not overdesign this phase; the main goal here is correct sequencing, not a full new taxonomy.
|
||||
|
||||
Acceptance:
|
||||
|
||||
- `role_span_profiles.json` is built from normalized structure, not pre-normalized seeds
|
||||
- backmatter/reference/body families are less stale
|
||||
|
||||
### Task 5: Upgrade rescue to use normalized structure as context
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_document.py`
|
||||
- `paperforge/worker/ocr_blocks.py`
|
||||
- tests in `tests/test_ocr_document.py`
|
||||
|
||||
#### 5.1 Keep rescue narrow but make it structurally consistent
|
||||
|
||||
Current rescue is acceptable in scope, but must operate on:
|
||||
|
||||
- normalized blocks
|
||||
- normalized profiles
|
||||
- final section boundaries
|
||||
|
||||
It should continue to own:
|
||||
|
||||
- frontmatter noise rescued back to body
|
||||
- body promoted to reference in references zone
|
||||
- weak heading demoted to body
|
||||
|
||||
#### 5.2 Prevent rescue from fighting stale roles
|
||||
|
||||
Once Task 4 is done, rescue no longer compares against pre-boundary profiles.
|
||||
|
||||
That should especially improve:
|
||||
|
||||
- backmatter child stability
|
||||
- reference consistency
|
||||
- weak heading/body disambiguation
|
||||
|
||||
Acceptance:
|
||||
|
||||
- rescue uses normalized context only
|
||||
- no rescue step depends on pre-normalization role families
|
||||
|
||||
### Task 6: Make downstream products consume the same structure truth
|
||||
|
||||
Files:
|
||||
|
||||
- `paperforge/worker/ocr_rebuild.py`
|
||||
- `paperforge/worker/ocr.py`
|
||||
- `paperforge/worker/ocr_render.py`
|
||||
- possibly `ocr_health.py`, `ocr_index.py`
|
||||
|
||||
#### 6.1 Thread document-structure output through downstream calls
|
||||
|
||||
Where possible, pass:
|
||||
|
||||
- normalized structured blocks
|
||||
- document structure
|
||||
- normalized profiles
|
||||
|
||||
into downstream stages instead of forcing each layer to reconstruct assumptions.
|
||||
|
||||
Minimal requirement:
|
||||
|
||||
- render must not recompute structure
|
||||
|
||||
Optional but desirable:
|
||||
|
||||
- health/index may later write structure summaries for debugging
|
||||
|
||||
#### 6.2 Consider persisting document structure artifact
|
||||
|
||||
Add a derived artifact such as:
|
||||
|
||||
- `structure/document_structure.json`
|
||||
|
||||
Contents may include:
|
||||
|
||||
- `body_end_page`
|
||||
- `backmatter_start`
|
||||
- `references_start`
|
||||
- `spread_start`
|
||||
- `spread_end`
|
||||
- `backmatter_form`
|
||||
|
||||
This is useful for:
|
||||
|
||||
- debugging
|
||||
- reproducibility
|
||||
- making render/index/health easier to compare
|
||||
|
||||
Acceptance:
|
||||
|
||||
- same structural truth is visible and reusable downstream
|
||||
|
||||
## Testing Plan
|
||||
|
||||
### Unit / integration targets
|
||||
|
||||
1. `tests/test_ocr_pdf_spans.py`
|
||||
- mapped OCR-to-PDF coordinate extraction
|
||||
- graceful fallback
|
||||
|
||||
2. `tests/test_ocr_roles.py`
|
||||
- remove expectations that depend on hard page gates or hard font thresholds
|
||||
- keep seed-role expectations conservative
|
||||
|
||||
3. `tests/test_ocr_document.py`
|
||||
- boundary detection still works
|
||||
- backmatter normalization still works
|
||||
- rescue uses normalized roles
|
||||
|
||||
4. `tests/test_ocr_render_stabilization.py`
|
||||
- render no longer depends on its own structural mutation
|
||||
- tail ordering remains correct when given normalized blocks
|
||||
|
||||
5. if needed, add a new test around `build_structured_blocks()` ordering
|
||||
- asserts profile build happens after document normalization
|
||||
|
||||
### Real-paper regression set
|
||||
|
||||
Must re-run on:
|
||||
|
||||
- `7C8829BD`
|
||||
- Frontiers-style mixed tail spread
|
||||
- `2GN9LMCW`
|
||||
- PeerJ-style declarations container + references + unnumbered headings
|
||||
|
||||
Verification focus:
|
||||
|
||||
- structure layer and rendered markdown agree on backmatter order
|
||||
- references remain stable
|
||||
- no reintroduced page marker mismatch
|
||||
- span metadata is present where expected and not obviously empty/misaligned
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
This plan is complete when all of the following are true:
|
||||
|
||||
1. `span_metadata` extraction no longer assumes OCR bbox == PDF coordinates
|
||||
2. `ocr_render.py` does not recompute or mutate document structure semantics
|
||||
3. `build_structured_blocks()` builds profiles after structural normalization
|
||||
4. rescue operates on normalized roles and normalized profiles
|
||||
5. `7C8829BD` and `2GN9LMCW` remain correct after rebuild
|
||||
6. tests cover the new sequencing and coordinate-baseline assumptions
|
||||
|
||||
## Risks To Watch
|
||||
|
||||
1. Over-coupling render to a new structure artifact too early
|
||||
- keep the first change minimal: pass normalized blocks first, persist structure later if useful
|
||||
|
||||
2. Breaking old tests that implicitly relied on hard threshold behavior
|
||||
- update tests toward seed-role behavior, not old heuristic internals
|
||||
|
||||
3. Span extraction still returning sparse data on scan-heavy PDFs
|
||||
- that is acceptable if graceful degradation remains intact
|
||||
- the important fix is to stop reading obviously wrong regions
|
||||
|
||||
## Recommended Execution Order
|
||||
|
||||
1. Task 1 — span coordinate fix
|
||||
2. Task 3 — make `ocr_document.py` the structure owner
|
||||
3. Task 4 — reorder profile construction
|
||||
4. Task 5 — rescue on normalized context
|
||||
5. Task 6 — downstream truth convergence
|
||||
6. finalize tests and real-paper rebuild verification
|
||||
|
||||
|
|
@ -591,26 +591,6 @@ def assign_block_role(
|
|||
evidence=[f"citation line pattern: {text[:60]}"],
|
||||
)
|
||||
|
||||
# Author list with superscript affiliation markers → authors, not noise
|
||||
# Distinguish from citation lines (which have year in parens) and
|
||||
# affiliation blocks (which have institutional keywords)
|
||||
has_year_parens = bool(re.search(r"\(\d{4}[a-z]?\)", text))
|
||||
has_inst_keyword = any(
|
||||
kw in lower_txt for kw in ["department", "university", "institute", "college", "school of"]
|
||||
)
|
||||
if (
|
||||
_AUTHOR_AFFILIATION_MARKER.search(text)
|
||||
and "," in text
|
||||
and not has_year_parens
|
||||
and not has_inst_keyword
|
||||
and len(text) < 500
|
||||
):
|
||||
return RoleAssignment(
|
||||
role="authors",
|
||||
confidence=0.8,
|
||||
evidence=[f"author list with affiliation markers: {text[:60]}"],
|
||||
)
|
||||
|
||||
# Affiliation block starting with superscript (page 1 only)
|
||||
if (
|
||||
still_frontmatter
|
||||
|
|
|
|||
|
|
@ -343,6 +343,31 @@ def test_heading_level_from_profile_match() -> None:
|
|||
pass # Placeholder — will be updated after refactor
|
||||
|
||||
|
||||
def test_body_citation_not_authors() -> None:
|
||||
from paperforge.worker.ocr_roles import assign_block_role
|
||||
block = {
|
||||
"block_label": "text",
|
||||
"block_content": "In Section 5, the focus is on ES based bioelectronics $^{8,49}$",
|
||||
"block_bbox": [100, 500, 800, 540],
|
||||
"page": 3,
|
||||
}
|
||||
result = assign_block_role(block, page_blocks=[block], page_width=1200, page_height=1600)
|
||||
assert result.role == "body_paragraph", f"Expected body_paragraph, got {result.role}"
|
||||
assert result.confidence >= 0.5
|
||||
|
||||
|
||||
def test_frontmatter_author_zone_still_works() -> None:
|
||||
from paperforge.worker.ocr_roles import assign_block_role
|
||||
block = {
|
||||
"block_label": "text",
|
||||
"block_content": "Alice Smith, Bob Jones, Charlie Brown",
|
||||
"block_bbox": [100, 300, 800, 330],
|
||||
"page": 1,
|
||||
}
|
||||
result = assign_block_role(block, page_blocks=[block], page_width=1200, page_height=1600)
|
||||
assert result.role == "authors", f"Expected authors, got {result.role}"
|
||||
|
||||
|
||||
def test_backmatter_boundary_detects_on_early_page() -> None:
|
||||
"""Backmatter boundary should be detectable on papers with fewer
|
||||
than 8 pages, without a hard page gate."""
|
||||
|
|
|
|||
Loading…
Reference in a new issue