mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: add relational-operator spacing and Greek-letter preservation to math normalization
This commit is contained in:
parent
cdf7deffc9
commit
c56ce87a1c
2 changed files with 62 additions and 6 deletions
|
|
@ -18,6 +18,13 @@ def normalize_inline_math_delimiters(text: str) -> str:
|
|||
return text
|
||||
|
||||
|
||||
_RELOP_PATTERN = re.compile(
|
||||
r'(?:\\geq|\\leq|\\sim|\\approx|\\gg|\\ll|>|<)\s*$'
|
||||
)
|
||||
|
||||
_GREEK_LETTER = re.compile(r'\\(?:[a-zA-Z]+)')
|
||||
|
||||
|
||||
def normalize_math_prose_boundaries(text: str) -> str:
|
||||
result: list[str] = []
|
||||
dollar_count = 0
|
||||
|
|
@ -31,14 +38,43 @@ def normalize_math_prose_boundaries(text: str) -> str:
|
|||
if ch == '$':
|
||||
dollar_count += 1
|
||||
is_opening = (dollar_count % 2 == 1)
|
||||
if is_opening and i > 0 and text[i - 1].isalnum() and text[i - 1] not in '-–—':
|
||||
result.append(' $')
|
||||
if is_opening:
|
||||
if i > 0 and text[i - 1].isalnum() and text[i - 1] not in '-–—':
|
||||
result.append(' $')
|
||||
else:
|
||||
result.append('$')
|
||||
else:
|
||||
result.append('$')
|
||||
else:
|
||||
result.append(ch)
|
||||
i += 1
|
||||
return ''.join(result)
|
||||
|
||||
text = ''.join(result)
|
||||
|
||||
def _after_space(m: re.Match) -> str:
|
||||
content = m.group(1)
|
||||
nxt = m.group(2)
|
||||
|
||||
if nxt.isdigit():
|
||||
return f"${content}$ {nxt}"
|
||||
|
||||
if _RELOP_PATTERN.search(content):
|
||||
return f"${content}$ {nxt}"
|
||||
|
||||
if _GREEK_LETTER.fullmatch(content.strip()):
|
||||
return f"${content}${nxt}"
|
||||
|
||||
if nxt.islower():
|
||||
return f"${content}$ {nxt}"
|
||||
|
||||
return f"${content}${nxt}"
|
||||
|
||||
text = re.sub(
|
||||
r'(?<!\$)\$([^$]+?)\$(?!\$)([a-zA-Z0-9])',
|
||||
_after_space,
|
||||
text,
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def normalize_display_math(text: str) -> str:
|
||||
|
|
|
|||
|
|
@ -649,7 +649,7 @@ def test_stabilize_tail_zone_references_kept_separate() -> None:
|
|||
def test_normalize_ocr_math_text_delimiter_spacing() -> None:
|
||||
from paperforge.worker.ocr_math import normalize_ocr_math_text
|
||||
assert normalize_ocr_math_text("$ ^{8,49} $") == "$^{8,49}$"
|
||||
assert normalize_ocr_math_text("$ \\\\mu $m") == "$\\\\mu$m"
|
||||
assert normalize_ocr_math_text(r"$ \mu $m") == r"$\mu$m"
|
||||
|
||||
|
||||
def test_normalize_ocr_math_text_citation_superscript() -> None:
|
||||
|
|
@ -666,8 +666,28 @@ def test_normalize_ocr_math_text_prose_boundary() -> None:
|
|||
|
||||
def test_normalize_ocr_math_text_hyphen_preserved() -> None:
|
||||
from paperforge.worker.ocr_math import normalize_ocr_math_text
|
||||
result = normalize_ocr_math_text("TGF-$\\\\beta$")
|
||||
assert "TGF-$\\\\beta$" in result
|
||||
result = normalize_ocr_math_text(r"TGF-$\beta$")
|
||||
assert r"TGF-$\beta$" in result
|
||||
|
||||
|
||||
def test_normalize_ocr_math_text_relop_followed_by_digit() -> None:
|
||||
from paperforge.worker.ocr_math import normalize_ocr_math_text
|
||||
assert normalize_ocr_math_text(r"$\geq$100") == r"$\geq$ 100"
|
||||
assert normalize_ocr_math_text(r"$\leq$0.5") == r"$\leq$ 0.5"
|
||||
assert normalize_ocr_math_text(r"$\sim$1000") == r"$\sim$ 1000"
|
||||
assert normalize_ocr_math_text(r"$\approx$3.14") == r"$\approx$ 3.14"
|
||||
|
||||
|
||||
def test_normalize_ocr_math_text_relop_followed_by_unit() -> None:
|
||||
from paperforge.worker.ocr_math import normalize_ocr_math_text
|
||||
result = normalize_ocr_math_text(r"$\sim$1000Scm^{-1}")
|
||||
assert r"$\sim$ 1000Scm^{-1}" in result
|
||||
|
||||
|
||||
def test_normalize_ocr_math_text_greek_letter_compound_preserved() -> None:
|
||||
from paperforge.worker.ocr_math import normalize_ocr_math_text
|
||||
assert normalize_ocr_math_text(r"NF-$\kappa$B") == r"NF-$\kappa$B"
|
||||
assert normalize_ocr_math_text(r"TGF-$\beta$") == r"TGF-$\beta$"
|
||||
|
||||
|
||||
def test_normalize_ocr_math_text_display_math() -> None:
|
||||
|
|
|
|||
Loading…
Reference in a new issue