mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
feat: add ocr_redo frontmatter field + sync preservation
This commit is contained in:
Research Assistant
parent
6eb0e2a2a1
commit
fb646a1129
4 changed files with 176 additions and 1 deletions
|
|
@ -380,6 +380,21 @@ def extract_preserved_tags(text: str) -> list[str] | None:
|
|||
return None
|
||||
|
||||
|
||||
def extract_preserved_ocr_redo(text: str) -> bool:
|
||||
"""Extract ocr_redo flag from existing note frontmatter.
|
||||
|
||||
Returns True if ocr_redo is explicitly set to true in YAML frontmatter.
|
||||
Returns False otherwise (missing, unreadable, or explicitly false).
|
||||
"""
|
||||
data = read_frontmatter_dict(text)
|
||||
val = data.get("ocr_redo", False)
|
||||
if isinstance(val, bool):
|
||||
return val
|
||||
if isinstance(val, str):
|
||||
return val.strip().lower() in ("true", "yes", "1")
|
||||
return False
|
||||
|
||||
|
||||
def read_frontmatter_dict(text: str) -> dict:
|
||||
"""Parse YAML frontmatter from markdown text using yaml.safe_load.
|
||||
|
||||
|
|
|
|||
|
|
@ -128,6 +128,13 @@ frontmatter:
|
|||
owner: ocr
|
||||
default: "none"
|
||||
introduced_in: "2.1"
|
||||
ocr_redo:
|
||||
type: bool
|
||||
required: false
|
||||
public: true
|
||||
description: "User requests OCR redo"
|
||||
owner: user
|
||||
introduced_in: "1.5.14"
|
||||
deep_reading_status:
|
||||
type: enum
|
||||
values:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from paperforge.adapters.obsidian_frontmatter import (
|
|||
canonicalize_decision,
|
||||
compute_final_collection,
|
||||
extract_preserved_deep_reading,
|
||||
extract_preserved_ocr_redo,
|
||||
extract_preserved_tags,
|
||||
update_frontmatter_field,
|
||||
)
|
||||
|
|
@ -995,10 +996,12 @@ def next_key(domain: str, export_rows: list[dict]) -> str:
|
|||
return f"{prefix}{max_num + 1:03d}"
|
||||
|
||||
|
||||
def frontmatter_note(entry: dict, existing_text: str = "", preserved_tags: list[str] | None = None) -> str:
|
||||
def frontmatter_note(entry: dict, existing_text: str = "", preserved_tags: list[str] | None = None, preserved_ocr_redo: bool | None = None) -> str:
|
||||
preserved_deep = extract_preserved_deep_reading(existing_text)
|
||||
if preserved_tags is None and existing_text:
|
||||
preserved_tags = extract_preserved_tags(existing_text)
|
||||
if preserved_ocr_redo is None and existing_text:
|
||||
preserved_ocr_redo = extract_preserved_ocr_redo(existing_text)
|
||||
first_author = entry.get("first_author", "")
|
||||
if not first_author:
|
||||
authors = entry.get("authors", [])
|
||||
|
|
@ -1033,6 +1036,8 @@ def frontmatter_note(entry: dict, existing_text: str = "", preserved_tags: list[
|
|||
f"fulltext_md_path: {yaml_quote('[[{}]]'.format(entry['fulltext_path']) if entry.get('ocr_status') == 'done' and entry.get('fulltext_path') else '')}",
|
||||
]
|
||||
)
|
||||
if preserved_ocr_redo:
|
||||
lines.append("ocr_redo: true")
|
||||
if preserved_tags is not None:
|
||||
lines.append("tags:")
|
||||
for tag in preserved_tags:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,39 @@ from unittest.mock import patch
|
|||
from PIL import Image
|
||||
|
||||
|
||||
def test_extract_preserved_ocr_redo_preserves_flag_across_frontmatter_rewrite() -> None:
|
||||
"""Simulate sync preserving ocr_redo through frontmatter_note()."""
|
||||
from paperforge.adapters.obsidian_frontmatter import extract_preserved_ocr_redo
|
||||
from paperforge.worker.sync import frontmatter_note
|
||||
|
||||
entry = {"zotero_key": "TEST", "title": "Test", "year": "2025"}
|
||||
|
||||
original = """---
|
||||
zotero_key: TEST
|
||||
title: Test
|
||||
year: 2025
|
||||
ocr_redo: true
|
||||
---
|
||||
"""
|
||||
preserved = extract_preserved_ocr_redo(original)
|
||||
assert preserved is True
|
||||
|
||||
rewritten = frontmatter_note(entry, existing_text=original)
|
||||
assert "ocr_redo: true" in rewritten
|
||||
|
||||
|
||||
def test_extract_preserved_ocr_redo_false_by_default() -> None:
|
||||
from paperforge.adapters.obsidian_frontmatter import extract_preserved_ocr_redo
|
||||
|
||||
text = """---
|
||||
zotero_key: TEST
|
||||
title: Test
|
||||
---
|
||||
"""
|
||||
assert extract_preserved_ocr_redo(text) is False
|
||||
|
||||
|
||||
|
||||
def test_caption_group_assignments_respects_columns() -> None:
|
||||
from paperforge.worker.ocr import caption_group_assignments
|
||||
|
||||
|
|
@ -256,6 +289,121 @@ def test_render_page_blocks_links_media_for_text_caption(tmp_path: Path) -> None
|
|||
assert any(line.startswith("Figure 4 RT-qPCR results.") for line in rendered)
|
||||
|
||||
|
||||
def test_render_page_blocks_keeps_author_year_references_as_reference_items(tmp_path: Path) -> None:
|
||||
from paperforge.worker.ocr import render_page_blocks
|
||||
|
||||
vault = tmp_path / "vault"
|
||||
images_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "images"
|
||||
page_cache_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "pages"
|
||||
images_dir.mkdir(parents=True)
|
||||
page_cache_dir.mkdir(parents=True)
|
||||
|
||||
result = {
|
||||
"prunedResult": {
|
||||
"width": 1200,
|
||||
"height": 1600,
|
||||
"parsing_res_list": [
|
||||
{"block_id": 1, "block_label": "paragraph_title", "block_bbox": [90, 120, 330, 160], "block_content": "References"},
|
||||
{
|
||||
"block_id": 2,
|
||||
"block_label": "reference_content",
|
||||
"block_bbox": [90, 220, 520, 300],
|
||||
"block_content": "Amin, B., Elahi, M. A., and Porter, E. (2019). A review of dielectric properties.",
|
||||
},
|
||||
{
|
||||
"block_id": 3,
|
||||
"block_label": "reference_content",
|
||||
"block_bbox": [620, 220, 1100, 300],
|
||||
"block_content": "Barker, A. T., and Lunt, M. J. (1983). The effects of pulsed magnetic fields.",
|
||||
},
|
||||
],
|
||||
},
|
||||
"inputImage": "",
|
||||
}
|
||||
|
||||
rendered = render_page_blocks(vault, 22, result, images_dir, page_cache_dir, pdf_doc=None)
|
||||
|
||||
assert "### References" in rendered
|
||||
assert any(line.startswith("Amin, B., Elahi") for line in rendered)
|
||||
assert any(line.startswith("Barker, A. T.") for line in rendered)
|
||||
|
||||
|
||||
def test_render_page_blocks_does_not_mix_tail_sections_into_references(tmp_path: Path) -> None:
|
||||
from paperforge.worker.ocr import render_page_blocks
|
||||
|
||||
vault = tmp_path / "vault"
|
||||
images_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "images"
|
||||
page_cache_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "pages"
|
||||
images_dir.mkdir(parents=True)
|
||||
page_cache_dir.mkdir(parents=True)
|
||||
|
||||
result = {
|
||||
"prunedResult": {
|
||||
"width": 1200,
|
||||
"height": 1600,
|
||||
"parsing_res_list": [
|
||||
{"block_id": 1, "block_label": "paragraph_title", "block_bbox": [90, 120, 330, 160], "block_content": "References"},
|
||||
{
|
||||
"block_id": 2,
|
||||
"block_label": "reference_content",
|
||||
"block_bbox": [90, 220, 520, 300],
|
||||
"block_content": "Amin, B., Elahi, M. A., and Porter, E. (2019). A review of dielectric properties.",
|
||||
},
|
||||
{
|
||||
"block_id": 3,
|
||||
"block_label": "paragraph_title",
|
||||
"block_bbox": [620, 120, 1080, 160],
|
||||
"block_content": "Generative AI statement",
|
||||
},
|
||||
{
|
||||
"block_id": 4,
|
||||
"block_label": "text",
|
||||
"block_bbox": [620, 220, 1100, 280],
|
||||
"block_content": "The author(s) declare that no Generative AI was used in the creation of this manuscript.",
|
||||
},
|
||||
],
|
||||
},
|
||||
"inputImage": "",
|
||||
}
|
||||
|
||||
rendered = render_page_blocks(vault, 22, result, images_dir, page_cache_dir, pdf_doc=None)
|
||||
refs_index = rendered.index("### References")
|
||||
ai_index = rendered.index("### Generative AI statement")
|
||||
|
||||
assert rendered[refs_index + 1].startswith("Amin, B., Elahi")
|
||||
assert rendered[ai_index + 1].startswith("The author(s) declare")
|
||||
|
||||
|
||||
def test_render_page_blocks_orders_references_within_column(tmp_path: Path) -> None:
|
||||
from paperforge.worker.ocr import render_page_blocks
|
||||
|
||||
vault = tmp_path / "vault"
|
||||
images_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "images"
|
||||
page_cache_dir = vault / "System" / "PaperForge" / "ocr" / "KEY" / "pages"
|
||||
images_dir.mkdir(parents=True)
|
||||
page_cache_dir.mkdir(parents=True)
|
||||
|
||||
result = {
|
||||
"prunedResult": {
|
||||
"width": 1200,
|
||||
"height": 1600,
|
||||
"parsing_res_list": [
|
||||
{"block_id": 1, "block_label": "paragraph_title", "block_bbox": [90, 120, 330, 160], "block_content": "References"},
|
||||
{"block_id": 2, "block_label": "reference_content", "block_bbox": [620, 200, 1100, 250], "block_content": "Bagnato, G. L. (2016)."},
|
||||
{"block_id": 3, "block_label": "reference_content", "block_bbox": [90, 200, 520, 250], "block_content": "Amin, B. (2019)."},
|
||||
{"block_id": 4, "block_label": "reference_content", "block_bbox": [90, 300, 520, 350], "block_content": "Cai, J. (2018)."},
|
||||
{"block_id": 5, "block_label": "reference_content", "block_bbox": [620, 300, 1100, 350], "block_content": "Bentley, G. (2012)."},
|
||||
],
|
||||
},
|
||||
"inputImage": "",
|
||||
}
|
||||
|
||||
rendered = render_page_blocks(vault, 22, result, images_dir, page_cache_dir, pdf_doc=None)
|
||||
ref_items = [line for line in rendered if line.startswith(("Amin", "Cai", "Bagnato", "Bentley"))]
|
||||
|
||||
assert ref_items == ["Amin, B. (2019).", "Cai, J. (2018).", "Bagnato, G. L. (2016).", "Bentley, G. (2012)."]
|
||||
|
||||
|
||||
def test_embedded_figure_text_excludes_body_paragraph_like_text() -> None:
|
||||
from paperforge.worker.ocr import is_embedded_figure_text_block
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue