obsidian-plugin/lllin000_PaperForge
1
0
Fork 0
mirror of https://github.com/lllin000/PaperForge.git synced 2026-07-22 06:50:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: remove preproof page 1 entirely instead of suppressing roles

Browse source
This commit is contained in:
Research Assistant 2026-06-16 14:15:21 +08:00
parent 9f5cb9095c
commit 2155197b87
3 changed files with 4 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
paperforge/worker/ocr_blocks.py
View file

@ -173,8 +173,8 @@ def build_structured_blocks(
rows.append(row)
# Suppress entire pre-proof page: if page 1 contains a pre-proof noise marker,
# mark all page-1 blocks as frontmatter_noise so the cover page is skipped
# Remove entire pre-proof page 1: if page 1 contains a pre-proof noise marker,
# filter out all page-1 blocks so the cover page is excluded from output entirely
from paperforge.worker.ocr_roles import is_preproof_marker
page1_has_preproof = any(
@ -183,11 +183,7 @@ def build_structured_blocks(
for row in rows
)
if page1_has_preproof:
for row in rows:
if (row.get("page", 0) or 0) == 1:
row["role"] = "frontmatter_noise"
row["render_default"] = False
row["index_default"] = False
rows = [row for row in rows if (row.get("page", 0) or 0) != 1]
body_family_anchor = discover_body_family_anchor(rows, page_count=total_pages)
doc_structure = DocumentStructure(body_family_anchor=body_family_anchor)

14
tests/fixtures/ocr_real_papers/DWQQK2YB/block_trace.csv vendored
View file

@ -1,18 +1,4 @@
page,block_id,raw_label,content_preview,bbox,role,role_confidence,evidence,seed_role,seed_confidence,zone,style_family,marker_type,render_default,index_default
1,0,paragraph_title,Journal Pre-proof,"[190, 206, 475, 246]",frontmatter_noise,0.98,"[""journal pre-proof marker: page 1, paragraph_title, y=206/1584, width=285/1224""]",frontmatter_noise,0.98,preproof_cover_zone,heading_like,preproof_marker,False,False
1,1,text,Magnetoresponsive Stem Cell Spheroid-based Cartilage Recovery Platform Utilizing Electromagnetic Fields,"[189, 314, 784, 365]",unknown_structural,0.8,"[""page-1 zone author_zone: Magnetoresponsive Stem Cell Spheroid-based Cartilage Recover""]",authors,0.8,frontmatter_main_zone,support_like,none,False,True
1,2,text,"Ami Yoo, Gwangjun Go, Kim Tien Nguyen, Kyungmin Lee, Hyun-Ki Min, Byungjeon Kang, Chang-Sei Kim, Jiwon Han, Jong-Oh Park, Eunpyo Choi","[190, 391, 783, 467]",authors,0.8,"[""page-1 zone author_zone: Ami Yoo, Gwangjun Go, Kim Tien Nguyen, Kyungmin Lee, Hyun-Ki""]",authors,0.8,frontmatter_main_zone,support_like,none,True,True
1,3,image,,"[846, 209, 1015, 436]",frontmatter_noise,0.85,"[""media label: image""]",media_asset,0.85,frontmatter_main_zone,support_like,empty,False,False
1,4,text,PII: S0925-4005(19)31768-X,"[190, 511, 632, 536]",frontmatter_noise,0.6,"[""default body_paragraph for text label""]",body_paragraph,0.6,frontmatter_main_zone,support_like,none,False,False
1,5,text,DOI: https://doi.org/10.1016/j.snb.2019.127569,"[190, 548, 778, 574]",frontmatter_noise,0.8,"[""page-1 zone journal_furniture_zone: DOI: https://doi.org/10.1016/j.snb.2019.127569""]",frontmatter_noise,0.8,,unknown_like,none,False,False
1,6,text,Reference: SNB 127569,"[191, 586, 526, 612]",frontmatter_noise,0.6,"[""default body_paragraph for text label""]",body_paragraph,0.6,frontmatter_main_zone,support_like,none,False,False
1,7,text,To appear in: Sensors and Actuators: B. Chemical,"[190, 651, 735, 676]",frontmatter_noise,0.6,"[""default body_paragraph for text label""]",body_paragraph,0.6,frontmatter_main_zone,support_like,none,False,False
1,8,text,Received Date: 13 August 2019,"[190, 713, 553, 739]",frontmatter_noise,0.7,"[""frontmatter noise text: Received Date: 13 August 2019""]",frontmatter_noise,0.7,frontmatter_main_zone,support_like,none,False,False
1,9,text,Revised Date: 19 November 2019,"[190, 751, 582, 777]",frontmatter_noise,0.6,"[""default body_paragraph for text label""]",body_paragraph,0.6,frontmatter_main_zone,support_like,none,False,False
1,10,text,Accepted Date: 10 December 2019,"[190, 789, 583, 814]",frontmatter_noise,0.8,"[""page-1 zone journal_furniture_zone: Accepted Date: 10 December 2019""]",frontmatter_noise,0.8,frontmatter_main_zone,support_like,none,False,False
1,11,text,"Please cite this article as: Yoo A, Go G, Nguyen KT, Lee K, Min H-Ki, Kang B, Kim C-Sei, Han J, Park J-Oh, Choi E, Magnetoresponsive Stem Cell Spheroid-based Cartilage Recovery Platform Utilizing Elec","[189, 876, 1019, 976]",frontmatter_noise,0.8,"[""page-1 zone journal_furniture_zone: Please cite this article as: Yoo A, Go G, Nguyen KT, Lee K, ""]",frontmatter_noise,0.8,,unknown_like,none,False,False
1,12,text,"This is a PDF file of an article that has undergone enhancements after acceptance, such as the addition of a cover page and metadata, and formatting for readability, but it is not yet the definitive v","[188, 1019, 1004, 1188]",frontmatter_noise,0.6,"[""default body_paragraph for text label""]",body_paragraph,0.6,frontmatter_main_zone,support_like,none,False,False
1,13,text,© 2019 Published by Elsevier.,"[189, 1209, 461, 1234]",frontmatter_noise,0.8,"[""page-1 zone journal_furniture_zone: \u00a9 2019 Published by Elsevier.""]",frontmatter_noise,0.8,frontmatter_main_zone,support_like,none,False,False
2,0,header,Journal Pre-proof,"[502, 1, 832, 40]",frontmatter_noise,0.98,"[""journal pre-proof running header suppressed: p2 y=1/1584""]",frontmatter_noise,0.98,preproof_cover_zone,unknown_like,preproof_marker,False,False
2,1,text,TITLE,"[76, 101, 146, 128]",unknown_structural,0.3,"[""short text, uncertain role""]",unknown_structural,0.3,frontmatter_side_zone,support_like,short_fragment,False,True
2,2,doc_title,"Magnetoresponsive Stem Cell Spheroid-based Cartilage Recovery

1 page block_id raw_label content_preview bbox role role_confidence evidence seed_role seed_confidence zone style_family marker_type render_default index_default
1 0 paragraph_title Journal Pre-proof [190, 206, 475, 246] frontmatter_noise 0.98 ["journal pre-proof marker: page 1, paragraph_title, y=206/1584, width=285/1224"] frontmatter_noise 0.98 preproof_cover_zone heading_like preproof_marker False False
1 1 text Magnetoresponsive Stem Cell Spheroid-based Cartilage Recovery Platform Utilizing Electromagnetic Fields [189, 314, 784, 365] unknown_structural 0.8 ["page-1 zone author_zone: Magnetoresponsive Stem Cell Spheroid-based Cartilage Recover"] authors 0.8 frontmatter_main_zone support_like none False True
1 2 text Ami Yoo, Gwangjun Go, Kim Tien Nguyen, Kyungmin Lee, Hyun-Ki Min, Byungjeon Kang, Chang-Sei Kim, Jiwon Han, Jong-Oh Park, Eunpyo Choi [190, 391, 783, 467] authors 0.8 ["page-1 zone author_zone: Ami Yoo, Gwangjun Go, Kim Tien Nguyen, Kyungmin Lee, Hyun-Ki"] authors 0.8 frontmatter_main_zone support_like none True True
1 3 image [846, 209, 1015, 436] frontmatter_noise 0.85 ["media label: image"] media_asset 0.85 frontmatter_main_zone support_like empty False False
1 4 text PII: S0925-4005(19)31768-X [190, 511, 632, 536] frontmatter_noise 0.6 ["default body_paragraph for text label"] body_paragraph 0.6 frontmatter_main_zone support_like none False False
1 5 text DOI: https://doi.org/10.1016/j.snb.2019.127569 [190, 548, 778, 574] frontmatter_noise 0.8 ["page-1 zone journal_furniture_zone: DOI: https://doi.org/10.1016/j.snb.2019.127569"] frontmatter_noise 0.8 unknown_like none False False
1 6 text Reference: SNB 127569 [191, 586, 526, 612] frontmatter_noise 0.6 ["default body_paragraph for text label"] body_paragraph 0.6 frontmatter_main_zone support_like none False False
1 7 text To appear in: Sensors and Actuators: B. Chemical [190, 651, 735, 676] frontmatter_noise 0.6 ["default body_paragraph for text label"] body_paragraph 0.6 frontmatter_main_zone support_like none False False
1 8 text Received Date: 13 August 2019 [190, 713, 553, 739] frontmatter_noise 0.7 ["frontmatter noise text: Received Date: 13 August 2019"] frontmatter_noise 0.7 frontmatter_main_zone support_like none False False
1 9 text Revised Date: 19 November 2019 [190, 751, 582, 777] frontmatter_noise 0.6 ["default body_paragraph for text label"] body_paragraph 0.6 frontmatter_main_zone support_like none False False
1 10 text Accepted Date: 10 December 2019 [190, 789, 583, 814] frontmatter_noise 0.8 ["page-1 zone journal_furniture_zone: Accepted Date: 10 December 2019"] frontmatter_noise 0.8 frontmatter_main_zone support_like none False False
1 11 text Please cite this article as: Yoo A, Go G, Nguyen KT, Lee K, Min H-Ki, Kang B, Kim C-Sei, Han J, Park J-Oh, Choi E, Magnetoresponsive Stem Cell Spheroid-based Cartilage Recovery Platform Utilizing Elec [189, 876, 1019, 976] frontmatter_noise 0.8 ["page-1 zone journal_furniture_zone: Please cite this article as: Yoo A, Go G, Nguyen KT, Lee K, "] frontmatter_noise 0.8 unknown_like none False False
1 12 text This is a PDF file of an article that has undergone enhancements after acceptance, such as the addition of a cover page and metadata, and formatting for readability, but it is not yet the definitive v [188, 1019, 1004, 1188] frontmatter_noise 0.6 ["default body_paragraph for text label"] body_paragraph 0.6 frontmatter_main_zone support_like none False False
1 13 text © 2019 Published by Elsevier. [189, 1209, 461, 1234] frontmatter_noise 0.8 ["page-1 zone journal_furniture_zone: \u00a9 2019 Published by Elsevier."] frontmatter_noise 0.8 frontmatter_main_zone support_like none False False
2 2 0 header Journal Pre-proof [502, 1, 832, 40] frontmatter_noise 0.98 ["journal pre-proof running header suppressed: p2 y=1/1584"] frontmatter_noise 0.98 preproof_cover_zone unknown_like preproof_marker False False
3 2 1 text TITLE [76, 101, 146, 128] unknown_structural 0.3 ["short text, uncertain role"] unknown_structural 0.3 frontmatter_side_zone support_like short_fragment False True
4 2 2 doc_title Magnetoresponsive Stem Cell Spheroid-based Cartilage Recovery Platform Utilizing Electromagnetic Fields [74, 155, 1028, 268] unknown_structural 0.2 ["unrecognized label 'doc_title'"] unknown_structural 0.2 body_zone body_like none False True

19
tests/fixtures/ocr_real_papers/DWQQK2YB/expectations.json vendored
View file

@ -8,23 +8,12 @@
"Updated expectations after RC3 (unassigned role) and RC4 (post-reference backmatter) fixes.",
"Figure captions as figure_caption_candidate is CORRECT per spec: object-gate roles stay as candidate at document-gate time.",
"Biographies now correctly in post_reference_backmatter_zone instead of reference_zone.",
"Title/authors on preproof page 1 still need frontmatter classification fix (RC1).",
"Page 1 (preproof cover) removed entirely from output.",
"OCR reads 'Amr Yoo' instead of 'Ami Yoo' on page 32.",
"Eunpyo Choi biography not present in OCR output."
]
},
"pages": {
"1": {
"assertions": [
{"text_contains": "Journal Pre-proof", "expected_role": "frontmatter_noise", "expected_zone": "preproof_cover_zone"},
{"text_contains": "Magnetoresponsive Stem Cell Spheroid", "expected_role": "paper_title", "notes": "BUG: Real: frontmatter_noise - should be paper_title"},
{"text_contains": "Ami Yoo, Gwangjun Go", "expected_role": "authors", "notes": "BUG: Real: frontmatter_noise - should be authors"},
{"text_contains": "PII:", "expected_role": "frontmatter_support", "notes": "BUG: Real: frontmatter_noise"},
{"text_contains": "To appear in:", "expected_role": "frontmatter_support", "notes": "BUG: Real: frontmatter_noise"},
{"text_contains": "Received Date:", "expected_role": "frontmatter_support", "notes": "BUG: Real: frontmatter_noise"},
{"text_contains": "2019 Published by Elsevier", "expected_role": "frontmatter_support", "notes": "BUG: Real: frontmatter_noise"}
]
},
"2": {
"assertions": [
{"text_contains": "Magnetoresponsive Stem Cell Spheroid", "expected_role": "paper_title", "notes": "BUG: Real: unknown_structural - should be paper_title"},
@ -205,12 +194,6 @@
}
},
"expected_bugs": [
{
"bug": "frontmatter_metadata_not_classified",
"pages": [1],
"description": "PII, DOI, To appear in, Received Date, Published by Elsevier have role=frontmatter_noise instead of frontmatter_support. Title/authors also noise.",
"fix": "Preproof page 1 frontmatter classification needs seed role rescue for title/authors/metadata lines"
},
{
"bug": "abstract_highlights_confused",
"pages": [3],