fix: add relational-operator spacing and Greek-letter preservation to math normalization

This commit is contained in:
Research Assistant 2026-06-06 17:19:33 +08:00
parent cdf7deffc9
commit c56ce87a1c
2 changed files with 62 additions and 6 deletions

View file

@ -18,6 +18,13 @@ def normalize_inline_math_delimiters(text: str) -> str:
return text
_RELOP_PATTERN = re.compile(
r'(?:\\geq|\\leq|\\sim|\\approx|\\gg|\\ll|>|<)\s*$'
)
_GREEK_LETTER = re.compile(r'\\(?:[a-zA-Z]+)')
def normalize_math_prose_boundaries(text: str) -> str:
result: list[str] = []
dollar_count = 0
@ -31,14 +38,43 @@ def normalize_math_prose_boundaries(text: str) -> str:
if ch == '$':
dollar_count += 1
is_opening = (dollar_count % 2 == 1)
if is_opening and i > 0 and text[i - 1].isalnum() and text[i - 1] not in '-–—':
result.append(' $')
if is_opening:
if i > 0 and text[i - 1].isalnum() and text[i - 1] not in '-–—':
result.append(' $')
else:
result.append('$')
else:
result.append('$')
else:
result.append(ch)
i += 1
return ''.join(result)
text = ''.join(result)
def _after_space(m: re.Match) -> str:
content = m.group(1)
nxt = m.group(2)
if nxt.isdigit():
return f"${content}$ {nxt}"
if _RELOP_PATTERN.search(content):
return f"${content}$ {nxt}"
if _GREEK_LETTER.fullmatch(content.strip()):
return f"${content}${nxt}"
if nxt.islower():
return f"${content}$ {nxt}"
return f"${content}${nxt}"
text = re.sub(
r'(?<!\$)\$([^$]+?)\$(?!\$)([a-zA-Z0-9])',
_after_space,
text,
)
return text
def normalize_display_math(text: str) -> str:

View file

@ -649,7 +649,7 @@ def test_stabilize_tail_zone_references_kept_separate() -> None:
def test_normalize_ocr_math_text_delimiter_spacing() -> None:
from paperforge.worker.ocr_math import normalize_ocr_math_text
assert normalize_ocr_math_text("$ ^{8,49} $") == "$^{8,49}$"
assert normalize_ocr_math_text("$ \\\\mu $m") == "$\\\\mu$m"
assert normalize_ocr_math_text(r"$ \mu $m") == r"$\mu$m"
def test_normalize_ocr_math_text_citation_superscript() -> None:
@ -666,8 +666,28 @@ def test_normalize_ocr_math_text_prose_boundary() -> None:
def test_normalize_ocr_math_text_hyphen_preserved() -> None:
from paperforge.worker.ocr_math import normalize_ocr_math_text
result = normalize_ocr_math_text("TGF-$\\\\beta$")
assert "TGF-$\\\\beta$" in result
result = normalize_ocr_math_text(r"TGF-$\beta$")
assert r"TGF-$\beta$" in result
def test_normalize_ocr_math_text_relop_followed_by_digit() -> None:
from paperforge.worker.ocr_math import normalize_ocr_math_text
assert normalize_ocr_math_text(r"$\geq$100") == r"$\geq$ 100"
assert normalize_ocr_math_text(r"$\leq$0.5") == r"$\leq$ 0.5"
assert normalize_ocr_math_text(r"$\sim$1000") == r"$\sim$ 1000"
assert normalize_ocr_math_text(r"$\approx$3.14") == r"$\approx$ 3.14"
def test_normalize_ocr_math_text_relop_followed_by_unit() -> None:
from paperforge.worker.ocr_math import normalize_ocr_math_text
result = normalize_ocr_math_text(r"$\sim$1000Scm^{-1}")
assert r"$\sim$ 1000Scm^{-1}" in result
def test_normalize_ocr_math_text_greek_letter_compound_preserved() -> None:
from paperforge.worker.ocr_math import normalize_ocr_math_text
assert normalize_ocr_math_text(r"NF-$\kappa$B") == r"NF-$\kappa$B"
assert normalize_ocr_math_text(r"TGF-$\beta$") == r"TGF-$\beta$"
def test_normalize_ocr_math_text_display_math() -> None: