lllin000_PaperForge/paperforge/worker/ocr.py

from __future__ import annotations

import html
import json
import logging
import os
import re
import shutil
from datetime import datetime, timezone
from json import JSONDecodeError
from pathlib import Path

import fitz
import requests
from PIL import Image

from paperforge.config import paperforge_paths
from paperforge.worker import sync as _sync
from paperforge.worker._progress import progress_bar


def _read_dotenv(vault: Path, key: str) -> str:
    """Read *key* from vault/.env or System/PaperForge/.env, return empty if not found."""
    for dotenv_path in (vault / ".env", paperforge_paths(vault).get("pipeline", Path()) / ".env"):
        try:
            for line in dotenv_path.read_text(encoding="utf-8").splitlines():
                line = line.strip()
                if line.startswith("#") or "=" not in line:
                    continue
                k, _, v = line.partition("=")
                if k.strip() == key:
                    return v.strip().strip('"').strip("'")
        except (OSError, UnicodeDecodeError):
            continue
    return ""


try:
    from paperforge.worker.ocr_roles import assign_block_role  # noqa: F401
except ImportError:
    pass

from paperforge.worker._retry import retry_with_meta
from paperforge.worker._utils import (
    pipeline_paths,
    read_json,
    write_json,
)
from paperforge.worker.asset_index import refresh_index_entry
from paperforge.worker.ocr_artifacts import (
    artifact_paths_for_key,
    build_version_payload,
    compute_json_hash,
    compute_pdf_fingerprint,
)
from paperforge.worker.ocr_blocks import (
    build_raw_blocks_for_result_lines,
    build_structured_blocks,
    write_raw_blocks_jsonl,
    write_structured_blocks_jsonl,
)
from paperforge.worker.ocr_figures import (
    build_figure_inventory,
    write_figure_inventory,
)
from paperforge.worker.ocr_metadata import (
    extract_frontmatter_candidates,
    resolve_metadata,
    write_resolved_metadata,
)
from paperforge.worker.ocr_tables import (
    build_table_inventory,
    write_table_inventory,
)
from paperforge.worker.sync import (
    load_control_actions,
    load_export_rows,
)

logger = logging.getLogger(__name__)


def ensure_ocr_meta(vault: Path, row: dict) -> dict:
    paths = pipeline_paths(vault)
    key = row["zotero_key"]
    meta_path = paths["ocr"] / key / "meta.json"
    meta_path.parent.mkdir(parents=True, exist_ok=True)
    try:
        meta = read_json(meta_path) if meta_path.exists() else {}
    except Exception:
        meta = {}
    meta.setdefault("zotero_key", key)
    meta.setdefault("source_pdf", row.get("pdf_path", ""))
    meta.setdefault("ocr_provider", "PaddleOCR-VL-1.6")
    meta.setdefault("mode", "async")
    meta.setdefault("ocr_status", "pending")
    meta.setdefault("ocr_job_id", "")
    meta.setdefault("ocr_started_at", "")
    meta.setdefault("ocr_finished_at", "")
    meta.setdefault("page_count", 0)
    meta.setdefault("markdown_path", "")
    meta.setdefault("json_path", "")
    try:
        assets_path = str((paths["ocr"] / key / "assets").relative_to(vault)).replace("\\", "/")
    except ValueError:
        assets_path = str(paths["ocr"] / key / "assets")
    meta.setdefault("assets_path", assets_path)
    meta.setdefault("fulltext_md_path", "")
    meta.setdefault("error", "")
    meta.setdefault("retry_count", 0)
    meta.setdefault("last_error", None)
    meta.setdefault("last_attempt_at", None)
    return meta


def _read_meta_or_empty(meta_path: Path) -> dict:
    try:
        return read_json(meta_path) if meta_path.exists() else {}
    except Exception:
        return {}


def validate_ocr_meta(paths: dict[str, Path], meta: dict) -> tuple[str, str]:
    status = str(meta.get("ocr_status", "pending") or "pending").strip().lower()
    if status != "done":
        return (status, str(meta.get("error", "") or ""))
    key = str(meta.get("zotero_key", "") or "").strip()
    if not key:
        return ("done_incomplete", "Missing zotero_key in OCR meta")
    ocr_root = paths["ocr"] / key
    fulltext_path = ocr_root / "fulltext.md"
    json_path = ocr_root / "json" / "result.json"
    page_count = int(meta.get("page_count", 0) or 0)
    if not fulltext_path.exists():
        return ("done_incomplete", "OCR fulltext.md missing")
    if not json_path.exists():
        return ("done_incomplete", "OCR result.json missing")
    fulltext_size = fulltext_path.stat().st_size
    json_size = json_path.stat().st_size
    if page_count < 1:
        return ("done_incomplete", "OCR page_count invalid")
    if fulltext_size < 500:
        return ("done_incomplete", "OCR fulltext.md too small")
    if json_size < 1000:
        return ("done_incomplete", "OCR result.json too small")
    try:
        rendered_pages = fulltext_path.read_text(encoding="utf-8").count("<!-- page ")
    except Exception:
        rendered_pages = 0
    if rendered_pages < 1:
        return ("done_incomplete", "OCR fulltext has no rendered pages")
    if rendered_pages != page_count:
        return ("done_incomplete", f"OCR page marker mismatch: meta={page_count}, rendered={rendered_pages}")
    return ("done", "")


def read_ocr_queue(paths: dict[str, Path]) -> list[dict]:
    queue_path = paths["ocr_queue"]
    if not queue_path.exists():
        return []
    try:
        data = read_json(queue_path)
        return data if isinstance(data, list) else []
    except JSONDecodeError:
        text = queue_path.read_text(encoding="utf-8")
        recovered = []
        decoder = json.JSONDecoder()
        index = 0
        while True:
            match = re.search("\\{", text[index:])
            if not match:
                break
            start = index + match.start()
            try:
                obj, end = decoder.raw_decode(text, start)
            except JSONDecodeError:
                index = start + 1
                continue
            if isinstance(obj, dict) and obj.get("zotero_key"):
                recovered.append(obj)
            index = end
        deduped = []
        seen = set()
        for row in recovered:
            key = row.get("zotero_key")
            if key in seen:
                continue
            seen.add(key)
            deduped.append(row)
        write_json(queue_path, deduped)
        return deduped


def write_ocr_queue(paths: dict[str, Path], queue_rows: list[dict]) -> None:
    deduped = []
    seen = set()
    for row in queue_rows:
        key = row.get("zotero_key")
        if not key or key in seen:
            continue
        seen.add(key)
        deduped.append(row)
    write_json(paths["ocr_queue"], deduped)


def sync_ocr_queue(paths: dict[str, Path], target_rows: list[dict]) -> list[dict]:
    existing_queue = read_ocr_queue(paths)
    target_map = {row["zotero_key"]: row for row in target_rows}
    synced_queue: list[dict] = []
    queued_keys: set[str] = set()
    for row in existing_queue:
        key = row.get("zotero_key")
        if not key or key not in target_map:
            continue
        target = target_map[key]
        if not target.get("has_pdf"):
            continue
        meta_path = paths["ocr"] / key / "meta.json"
        meta = _read_meta_or_empty(meta_path)
        status = str(meta.get("ocr_status", "pending") or "pending").strip().lower()
        if status in {"done", "blocked"}:
            continue
        if status == "nopdf":
            status = "pending"
        synced = dict(row)
        synced["has_pdf"] = bool(target.get("has_pdf"))
        synced["pdf_path"] = target.get("pdf_path", "")
        synced["queue_status"] = status
        if not synced.get("queued_at"):
            synced["queued_at"] = datetime.now(timezone.utc).isoformat()
        synced_queue.append(synced)
        queued_keys.add(key)
    now = datetime.now(timezone.utc).isoformat()
    for row in target_rows:
        key = row["zotero_key"]
        if key in queued_keys:
            continue
        if not row.get("has_pdf"):
            continue
        meta_path = paths["ocr"] / key / "meta.json"
        meta = _read_meta_or_empty(meta_path)
        status = str(meta.get("ocr_status", "pending") or "pending").strip().lower()
        if status in {"done", "blocked"}:
            continue
        if status == "nopdf":
            status = "pending"
            continue
        synced_queue.append(
            {
                "zotero_key": key,
                "has_pdf": bool(row.get("has_pdf")),
                "pdf_path": row.get("pdf_path", ""),
                "queued_at": now,
                "queue_status": status,
            }
        )
    write_ocr_queue(paths, synced_queue)
    return synced_queue


def cleanup_blocked_ocr_dirs(paths: dict[str, Path]) -> None:
    for meta_path in paths["ocr"].glob("*/meta.json"):
        try:
            meta = read_json(meta_path)
        except Exception:
            continue
        status = str(meta.get("ocr_status", "") or "").strip().lower()
        str(meta.get("zotero_key", "") or "").strip()
        ocr_dir = meta_path.parent
        if status != "blocked":
            continue
        has_payload = any(
            candidate.exists() for candidate in [ocr_dir / "fulltext.md", ocr_dir / "json" / "result.json"]
        )
        if has_payload:
            continue
        shutil.rmtree(ocr_dir, ignore_errors=True)


def normalize_obsidian_markdown(text: str) -> str:
    normalized = text.replace("\r\n", "\n")
    normalized = re.sub("[^\\S\\n]+\\$", "$", normalized)
    normalized = re.sub("\\$[^\\S\\n]+", "$", normalized)
    normalized = re.sub(
        "\\$\\^\\{([^}]+)\\*\\}\\$",
        lambda m: _superscript_to_correspondence_footnote(m.group(1)),
        normalized,
    )
    normalized = re.sub(
        "\\$\\^\\{([^}]*)\\\\dagger\\}\\$", lambda m: _superscript_to_equal_footnote(m.group(1)), normalized
    )
    normalized = re.sub("\\$\\^\\{([^}]*)†\\}\\$", lambda m: _superscript_to_equal_footnote(m.group(1)), normalized)
    normalized = re.sub("\\$\\s+\\^", "$^", normalized)
    normalized = re.sub("\\^\\{\\s+", "^{", normalized)
    normalized = re.sub("\\s+\\}", "}", normalized)
    normalized = normalized.replace("$^{ID}$", "")
    normalized = re.sub("[^\\S\\n]{2,}", " ", normalized)
    normalized = re.sub(
        "(?m)^\\*\\s*Correspondence:\\s*(.+)$", lambda match: f"[^correspondence]: {match.group(1).strip()}", normalized
    )
    normalized = re.sub(
        "(?m)^Correspondence:\\s*(.+)$", lambda match: f"[^correspondence]: {match.group(1).strip()}", normalized
    )
    normalized = re.sub("(?m)^†\\s*(.+)$", lambda match: f"[^equal]: {match.group(1).strip()}", normalized)
    normalized = re.sub(
        r"(?<!\\)(\*{1,3})\s*(?:=\s*)?(p\s*<\s*[\d]+(?:\.[\d]+)?)",
        lambda m: f"${m.group(1)}{m.group(2).strip()}$",
        normalized,
    )
    normalized = re.sub(
        r"\$\^\{\*\}\$\s*=\s*p\s*<\s*([\d]+(?:\.[\d]+)?)",
        lambda m: f"$*p < {m.group(1)}$",
        normalized,
    )
    normalized = re.sub(
        r"\$\^\{\*\}\$\s*(p\s*<\s*[\d]+(?:\.[\d]+)?)",
        lambda m: f"$*{m.group(1).strip()}$",
        normalized,
    )
    normalized = re.sub(
        r"(?<!\$)\bp\s*<\s*[\d]+(?:\.[\d]+)?",
        lambda m: f"${m.group(0).strip()}$" if normalized[: m.start()].count("$") % 2 == 0 else m.group(0),
        normalized,
    )
    normalized = re.sub("([A-Za-z])(\\$[^$\\n]+\\$)", "\\1 \\2", normalized)
    normalized = re.sub("(\\$[^$\\n]+\\$)([A-Za-z])", "\\1 \\2", normalized)
    normalized = re.sub("\\n{3,}", "\n\n", normalized)
    return normalized.strip()


def _image_embed_for_obsidian(vault_rel: Path) -> str:
    return f"![[{str(vault_rel).replace(chr(92), '/')}]]"


def _superscript_to_equal_footnote(content: str) -> str:
    cleaned = re.sub("(,)?\\\\dagger", "", content).replace("†", "").strip(", ")
    if cleaned:
        return f"$^{{{cleaned}}}$[^equal]"
    return "[^equal]"


def _superscript_to_correspondence_footnote(content: str) -> str:
    cleaned = content.replace("*", "").strip(", ")
    if cleaned:
        return f"$^{{{cleaned}}}$[^correspondence]"
    return "[^correspondence]"


def ensure_page_image_cached(url: str, destination: Path) -> Path | None:
    if destination.exists():
        return destination
    if not url:
        return None
    try:
        response = requests.get(url, timeout=120)
        response.raise_for_status()
    except requests.RequestException:
        return None
    destination.parent.mkdir(parents=True, exist_ok=True)
    destination.write_bytes(response.content)
    return destination


def render_pdf_page_cached(
    pdf_doc, page_index: int, target_width: int, target_height: int, destination: Path
) -> Path | None:
    if destination.exists():
        return destination
    if not pdf_doc or page_index < 1 or page_index > len(pdf_doc):
        return None
    try:
        page = pdf_doc[page_index - 1]
        rect = page.rect
        zoom_x = target_width / rect.width if target_width and rect.width else 2.0
        zoom_y = target_height / rect.height if target_height and rect.height else 2.0
        pix = page.get_pixmap(matrix=fitz.Matrix(zoom_x, zoom_y), alpha=False)
        destination.parent.mkdir(parents=True, exist_ok=True)
        pix.save(str(destination))
        return destination
    except Exception:
        return None


def crop_block_asset(page_image_path: Path, bbox: list[int], destination: Path) -> bool:
    if destination.exists():
        return True
    try:
        with Image.open(page_image_path) as img:
            x1, y1, x2, y2 = (max(0, int(v)) for v in bbox)
            if x2 <= x1 or y2 <= y1:
                return False
            crop = img.crop((x1, y1, x2, y2))
            destination.parent.mkdir(parents=True, exist_ok=True)
            crop.save(destination)
            return True
    except Exception:
        return False


def block_sort_key(block: dict) -> tuple:
    bbox = block.get("block_bbox", [0, 0, 0, 0])
    order = block.get("block_order")
    return (0 if order is not None else 1, order or 0, int(bbox[1]), int(bbox[0]))


def _col_xc(block: dict) -> float | None:
    bbox = block.get("block_bbox", [0, 0, 0, 0])
    if len(bbox) < 3 or bbox[2] <= bbox[0]:
        return None
    return (bbox[0] + bbox[2]) / 2


def _column_kind(block: dict, page_width: int) -> str | None:
    bbox = block.get("block_bbox", [0, 0, 0, 0])
    if len(bbox) < 4 or bbox[2] <= bbox[0]:
        return None
    mid = page_width / 2 if page_width else 600
    x1, _, x2, _ = bbox
    width = x2 - x1
    # Full-width or center-spanning blocks should keep their original slot.
    if (x1 < mid and x2 > mid) or (page_width and width >= page_width * 0.55):
        return "S"
    xc = (x1 + x2) / 2
    if xc < mid * 0.95:
        return "L"
    if xc > mid * 1.05:
        return "R"
    return "S"


def _col_switches(blocks: list[dict], page_width: int) -> int:
    switches = 0
    prev_col = None
    for b in blocks:
        col = _column_kind(b, page_width)
        if col is None:
            continue
        if prev_col and col != "S" and prev_col != "S" and col != prev_col:
            switches += 1
        if col != "S":
            prev_col = col
    return switches


def validate_block_order(blocks: list[dict], page_width: int) -> list[dict]:
    if len(blocks) < 4 or not page_width:
        return blocks

    if _col_switches(blocks, page_width) > 3:
        left = [b for b in blocks if _column_kind(b, page_width) == "L"]
        right = [b for b in blocks if _column_kind(b, page_width) == "R"]
        centered = [b for b in blocks if _column_kind(b, page_width) not in {"L", "R"}]
        left.sort(key=lambda b: b.get("block_bbox", [0, 0, 0, 0])[1])
        right.sort(key=lambda b: b.get("block_bbox", [0, 0, 0, 0])[1])
        if not centered:
            return left + right
        result: list[dict] = []
        iters = {"L": iter(left), "R": iter(right)}
        for b in blocks:
            col_key = _column_kind(b, page_width)
            if col_key not in {"L", "R"}:
                result.append(b)
                continue
            try:
                result.append(next(iters[col_key]))
            except StopIteration:
                result.append(b)
        return result

    col_buckets: dict[str, list[dict]] = {"L": [], "R": []}
    for b in blocks:
        col_key = _column_kind(b, page_width)
        if col_key not in {"L", "R"}:
            continue
        col_buckets[col_key].append(b)

    for key in ("L", "R"):
        sorted_col = sorted(col_buckets[key], key=lambda b: b.get("block_bbox", [0, 0, 0, 0])[1])
        if [id(blk) for blk in col_buckets[key]] != [id(blk) for blk in sorted_col]:
            col_buckets[key] = sorted_col

    result: list[dict] = []
    iters = {"L": iter(col_buckets["L"]), "R": iter(col_buckets["R"])}
    for b in blocks:
        col_key = _column_kind(b, page_width)
        if col_key not in {"L", "R"}:
            result.append(b)
            continue
        try:
            result.append(next(iters[col_key]))
        except StopIteration:
            pass
    return result


def clean_block_text(text: str) -> str:
    text = html.unescape(normalize_obsidian_markdown(text)).strip()
    return text


def is_subfigure_label(text: str) -> bool:
    compact = re.sub("\\s+", " ", text.strip().lower())
    return bool(
        re.fullmatch("(?:\\([a-z]\\)\\s*)+", compact)
        or re.fullmatch("[a-z]", compact)
        or re.fullmatch("[a-z]\\)", compact)
    )


def is_affiliation_line(text: str) -> bool:
    compact = " ".join(text.split())
    if re.match("^(?:\\$\\^\\{)?\\d+(?:[,\\d-]*)?(?:\\})?", compact):
        return True
    keywords = (
        "university",
        "department",
        "college",
        "school",
        "institute",
        "hospital",
        "center",
        "centre",
        "laboratory",
        "lab",
        "program",
        "research",
    )
    lower = compact.lower()
    return any(word in lower for word in keywords) and len(compact) > 30


def is_frontmatter_noise_line(text: str) -> bool:
    compact = " ".join(text.split())
    lower = compact.lower()
    if compact in {"Article", "Review"}:
        return True
    prefixes = (
        "academic editor:",
        "copyright:",
        "licensee",
        "this article is an open access article",
        "publisher's note:",
        "check for updates",
    )
    if lower.startswith(prefixes):
        return True
    if re.fullmatch("\\d+\\s+\\w{1,6}", compact):
        return True
    return bool(len(compact) < 18 and re.search("\\d", compact))


def is_reference_tail_noise_line(text: str) -> bool:
    compact = clean_block_text(text)
    lower = compact.lower()
    if "submit your next manuscript" in lower:
        return True
    patterns = (
        "^doi:\\s*10\\.",
        "^Cite this article as:",
        "^Submit your next manuscript",
        "^BioMed Central\\b",
        "^Convenient online submission$",
        "^Thorough peer review$",
        "^No space constraints",
        "^Immediate publication on acceptance$",
        "^Inclusion in PubMed, CAS, Scopus and Google Scholar$",
        "^Research which is freely available for redistribution$",
        "^[•\\-]\\s*(Convenient online submission|Thorough peer review|No space constraints|Immediate publication on acceptance|Inclusion in PubMed|Research which is freely available)",
    )
    return any(re.match(pattern, compact, flags=re.IGNORECASE) for pattern in patterns)


def parse_reference_number(text: str) -> int | None:
    text = clean_block_text(text)
    m = re.match("^\\s*(\\d+)[\\.\\)]\\s*", text)
    if m:
        return int(m.group(1))
    m = re.match("^\\s*\\[(\\d+)\\][\\]\\.\\)\\s]*", text)
    if m:
        return int(m.group(1))
    return None


def is_reference_lead_block(text: str) -> bool:
    text = clean_block_text(text)
    if not text:
        return False
    if parse_reference_number(text) is not None:
        return True
    if re.match(r"^\s*[^\n]{1,160}\(\d{4}[a-z]?\)\.", text):
        return True
    if re.match(r"^\s*[A-Z][A-Za-z'’\-]+(?:\s+[A-Z][A-Za-z'’\-]+)*(?:,\s|\s+et al\.)", text):
        return True
    return False


def _bbox_column_bucket_xc(block: dict) -> float | None:
    bbox = block.get("block_bbox", [0, 0, 0, 0])
    if bbox[2] <= bbox[0]:
        return None
    return (bbox[0] + bbox[2]) / 2


def sort_reference_blocks(blocks: list[dict]) -> list[dict]:
    numbered_count = sum(parse_reference_number(block.get("block_content", "")) is not None for block in blocks)
    if numbered_count >= max(3, len(blocks) // 2):
        return sorted(
            blocks,
            key=lambda block: (
                parse_reference_number(block.get("block_content", "")) is None,
                parse_reference_number(block.get("block_content", "")) or 10**9,
                block_sort_key(block),
            ),
        )
    xcs = [xc for block in blocks if (xc := _bbox_column_bucket_xc(block)) is not None]
    if not xcs:
        mid = 600
    else:
        mid = sum(xcs) / len(xcs)
    left: list[dict] = []
    right: list[dict] = []
    for block in blocks:
        xc = _bbox_column_bucket_xc(block)
        if xc is None or xc < mid - 60:
            left.append(block)
        else:
            right.append(block)
    y_sort = lambda block: int(block.get("block_bbox", [0, 0, 0, 0])[1])
    left.sort(key=y_sort)
    right.sort(key=y_sort)
    return left + right


def assign_reference_continuation(continuation_block: dict, reference_blocks: list[dict]) -> int | None:
    bbox = continuation_block.get("block_bbox", [0, 0, 0, 0])
    x1, y1, x2, _ = bbox
    candidates = []
    for index, ref_block in enumerate(reference_blocks):
        rb = ref_block.get("block_bbox", [0, 0, 0, 0])
        rx1, ry1, rx2, ry2 = rb
        horizontal_overlap = min(x2, rx2) - max(x1, rx1)
        same_column = horizontal_overlap > 0 or abs((x1 + x2) / 2 - (rx1 + rx2) / 2) < 180
        if not same_column or ry1 > y1:
            continue
        candidates.append((index, ry2, abs((x1 + x2) / 2 - (rx1 + rx2) / 2)))
    if not candidates:
        return None
    candidates.sort(key=lambda item: (-item[1], item[2]))
    return candidates[0][0]


def build_reference_section_lines(reference_blocks: list[dict], reference_continuations: list[dict]) -> list[str]:
    if not reference_blocks:
        return []
    lines = [""]
    ordered_reference_blocks = sort_reference_blocks(reference_blocks)
    continuation_map: dict[int, list[str]] = {}
    sorted_continuations = sorted(reference_continuations, key=block_sort_key)
    assigned_continuation_indexes: set[int] = set()
    incomplete_reference_indexes = [
        index
        for index, block in enumerate(ordered_reference_blocks)
        if clean_block_text(block.get("block_content", "")).rstrip().endswith(":")
    ]
    for index, continuation in zip(incomplete_reference_indexes, sorted_continuations, strict=False):
        continuation_text = clean_block_text(continuation.get("block_content", ""))
        if continuation_text:
            continuation_map.setdefault(index, []).append(continuation_text)
            assigned_continuation_indexes.add(id(continuation))
    for continuation in sorted_continuations:
        if id(continuation) in assigned_continuation_indexes:
            continue
        target_index = assign_reference_continuation(continuation, ordered_reference_blocks)
        if target_index is None:
            continue
        continuation_map.setdefault(target_index, []).append(clean_block_text(continuation.get("block_content", "")))
        assigned_continuation_indexes.add(id(continuation))
    unassigned_continuations = [
        clean_block_text(continuation.get("block_content", ""))
        for continuation in sorted_continuations
        if id(continuation) not in assigned_continuation_indexes
    ]
    for index, block in enumerate(ordered_reference_blocks):
        text = clean_block_text(block.get("block_content", ""))
        if text:
            lines.append(text)
        for continuation_text in continuation_map.get(index, []):
            if continuation_text:
                lines.append(continuation_text)
        if text.rstrip().endswith(":") and unassigned_continuations:
            lines.append(unassigned_continuations.pop(0))
    return lines


def append_reference_section(rendered: list[str], reference_blocks: list[dict], reference_continuations: list[dict]) -> None:
    rendered.extend(build_reference_section_lines(reference_blocks, reference_continuations))


def clean_author_line(text: str) -> str:
    text = clean_block_text(text)
    text = text.replace("$^{ID}$", "")
    text = re.sub("\\s*,\\s*", ", ", text)
    text = re.sub("\\s+and\\s+", " and ", text)
    text = re.sub("([a-z])and ([A-Z])", "\\1 and \\2", text)
    return text


def handle_first_page_metadata_lines(
    text: str, rendered: list[str], affiliation_buffer: list[str], deferred_meta: list[str], footnotes: list[str]
) -> bool:
    handled = False
    lines = [part.strip() for part in text.splitlines() if part.strip()]
    if not lines:
        return True
    for line in lines:
        if is_frontmatter_noise_line(line):
            handled = True
            continue
        if line.startswith("Citation:"):
            deferred_meta.append(line)
            handled = True
            continue
        if (
            line.startswith("Received:")
            or line.startswith("Revised:")
            or line.startswith("Accepted:")
            or line.startswith("Published:")
        ):
            deferred_meta.append(line)
            handled = True
            continue
        if line.startswith("Correspondence:") or line.startswith("* Correspondence:"):
            footnotes[:] = [fn for fn in footnotes if not fn.startswith("[^correspondence]:")]
            footnotes.append(f"[^correspondence]: {line.split(':', 1)[1].strip()}")
            handled = True
            continue
        if line.startswith("† "):
            footnotes[:] = [fn for fn in footnotes if not fn.startswith("[^equal]:")]
            footnotes.append(f"[^equal]: {line[2:].strip()}")
            handled = True
            continue
        if is_affiliation_line(line):
            affiliation_buffer.append(line)
            handled = True
            continue
        if "$^{" in line and "," in line and (" and " in line):
            rendered.append(clean_author_line(line))
            handled = True
            continue
        if len(line) < 22 or re.fullmatch("\\d+\\s+\\w+", line):
            handled = True
            continue
    return handled


def footnote_marker_and_body(content: str) -> tuple[str, str]:
    cleaned = clean_block_text(content)
    match = re.match("^\\$?\\^?\\{?([^\\}\\s]+)\\}?\\$?\\s*(.+)$", cleaned)
    if match:
        return (match.group(1).strip(), match.group(2).strip())
    return ("", cleaned)


def attach_footnote_reference(text: str, marker: str, footnote_id: str) -> str:
    if not marker:
        return text
    candidates = [f"$^{{{marker}}}$", f"$^{{{marker},*}}$", f"$^{{*,{marker}}}$"]
    for token in candidates:
        if token in text and f"{token}[^{footnote_id}]" not in text:
            return text.replace(token, f"{token}[^{footnote_id}]", 1)
    return text


def parse_vision_footnote_entries(content: str) -> list[tuple[str, str]]:
    cleaned = clean_block_text(content)
    if not cleaned:
        return []
    matches = re.findall("\\$?\\^\\{?([A-Za-z*]+)\\}?\\$?\\s*([^;]+)", cleaned)
    if matches:
        return [(marker.strip(), body.strip()) for marker, body in matches if body.strip()]
    marker, body = footnote_marker_and_body(content)
    return [(marker, body)] if body else []


def _parse_asset_bbox_from_line(line: str) -> tuple[int, int, int, int] | None:
    match = re.search("page_(\\d+)_(?:figure|table)_(\\d+)_(\\d+)_(\\d+)_(\\d+)\\.jpg", line)
    if not match:
        return None
    return tuple(int(match.group(i)) for i in range(2, 6))


def _bbox_area(bbox: tuple[int, int, int, int]) -> int:
    x1, y1, x2, y2 = bbox
    return max(0, x2 - x1) * max(0, y2 - y1)


def _bbox_contains(outer: tuple[int, int, int, int], inner: tuple[int, int, int, int], margin: int = 12) -> bool:
    ox1, oy1, ox2, oy2 = outer
    ix1, iy1, ix2, iy2 = inner
    return ox1 - margin <= ix1 and oy1 - margin <= iy1 and (ox2 + margin >= ix2) and (oy2 + margin >= iy2)


def dedupe_page_media_lines(lines: list[str]) -> list[str]:
    media_indexes: list[tuple[int, tuple[int, int, int, int]]] = []
    for idx, line in enumerate(lines):
        if not line.startswith("![["):
            continue
        bbox = _parse_asset_bbox_from_line(line)
        if bbox:
            media_indexes.append((idx, bbox))
    drop_indexes: set[int] = set()
    for idx, bbox in media_indexes:
        for other_idx, other_bbox in media_indexes:
            if idx == other_idx:
                continue
            if _bbox_contains(other_bbox, bbox) and _bbox_area(other_bbox) > _bbox_area(bbox) * 1.15:
                drop_indexes.add(idx)
                break
    return [line for i, line in enumerate(lines) if i not in drop_indexes]


def attach_table_footnotes(table_html: str, marker_to_id: dict[str, str]) -> str:
    def replacer(match):
        markers = [part.strip() for part in match.group(1).split(",")]
        refs = []
        for marker in markers:
            ref_id = marker_to_id.get(marker)
            refs.append(f"{marker}[^{ref_id}]" if ref_id else marker)
        return ", ".join(refs) + "</td>"

    return re.sub("([a-z](?:\\s*,\\s*[a-z])*)</td>", replacer, table_html)


def media_clusters(blocks: list[dict]) -> tuple[dict[int, int], list[list[dict]]]:
    media = [b for b in blocks if b.get("block_label") in {"image", "chart"}]
    clusters: list[list[dict]] = []
    block_to_cluster: dict[int, int] = {}
    for block in media:
        x1, y1, x2, y2 = block.get("block_bbox", [0, 0, 0, 0])
        assigned = None
        for idx, cluster in enumerate(clusters):
            cx1 = min(item["block_bbox"][0] for item in cluster) - 40
            cy1 = min(item["block_bbox"][1] for item in cluster) - 40
            cx2 = max(item["block_bbox"][2] for item in cluster) + 40
            cy2 = max(item["block_bbox"][3] for item in cluster) + 40
            vertical_overlap = max(0, min(y2, cy2) - max(y1, cy1))
            min_height = max(1, min(y2 - y1, cy2 - cy1))
            if x2 < cx1:
                horizontal_gap = cx1 - x2
            elif x1 > cx2:
                horizontal_gap = x1 - cx2
            else:
                horizontal_gap = 0
            overlaps_cluster = not (x2 < cx1 or x1 > cx2 or y2 < cy1 or y1 > cy2)
            side_by_side_panel = vertical_overlap >= int(min_height * 0.35) and horizontal_gap <= 60
            if overlaps_cluster or side_by_side_panel:
                assigned = idx
                break
        if assigned is None:
            assigned = len(clusters)
            clusters.append([])
        clusters[assigned].append(block)
        block_to_cluster[block.get("block_id", -1)] = assigned
    return (block_to_cluster, clusters)


def _bbox_width(bbox: list[int]) -> int:
    return max(0, int(bbox[2]) - int(bbox[0]))


def _bbox_height(bbox: list[int]) -> int:
    return max(0, int(bbox[3]) - int(bbox[1]))


def _bbox_horizontal_overlap(a: list[int], b: list[int]) -> int:
    return max(0, min(int(a[2]), int(b[2])) - max(int(a[0]), int(b[0])))


def _bbox_vertical_overlap(a: list[int], b: list[int]) -> int:
    return max(0, min(int(a[3]), int(b[3])) - max(int(a[1]), int(b[1])))


def _bbox_horizontal_overlap_ratio(a: list[int], b: list[int]) -> float:
    width = min(_bbox_width(a), _bbox_width(b))
    if width <= 0:
        return 0.0
    return _bbox_horizontal_overlap(a, b) / width


def _bbox_center_x(bbox: list[int]) -> float:
    return (int(bbox[0]) + int(bbox[2])) / 2


def _bbox_center_y(bbox: list[int]) -> float:
    return (int(bbox[1]) + int(bbox[3])) / 2


def _cluster_bbox(cluster: list[dict]) -> list[int]:
    return [
        min(int(item["block_bbox"][0]) for item in cluster),
        min(int(item["block_bbox"][1]) for item in cluster),
        max(int(item["block_bbox"][2]) for item in cluster),
        max(int(item["block_bbox"][3]) for item in cluster),
    ]


def _union_bboxes(bbox_list: list[list[int]]) -> list[int]:
    return [
        min(int(bbox[0]) for bbox in bbox_list),
        min(int(bbox[1]) for bbox in bbox_list),
        max(int(bbox[2]) for bbox in bbox_list),
        max(int(bbox[3]) for bbox in bbox_list),
    ]


def is_formal_figure_legend(text: str) -> bool:
    cleaned = clean_block_text(text)
    if not cleaned:
        return False
    return bool(
        re.match(
            "^(?:Extended\\s+Data\\s+Fig\\.?\\s+\\w+|Extended\\s+Data\\s+Figure\\s+\\w+|Extended\\s+Data\\s+Table\\s+\\w+|Supplementary\\s+Fig\\.?\\s+\\w+|Supplementary\\s+Figure\\s+\\w+|Supplementary\\s+Table\\s+\\w+|Supplementary\\s+Video\\s+\\w+|Figure\\s+\\d+|Fig\\.?\\s+\\d+|Table\\s+\\d+|Scheme\\s+\\w+|Graphical\\s+Abstract(?:\\s*[:|.\\-].*)?)",
            cleaned,
            flags=re.IGNORECASE,
        )
    )


def is_numbered_figure_caption(text: str) -> bool:
    cleaned = clean_block_text(text)
    if not re.match(r"^(?:Figure|Fig\.?)\s+\d+\b", cleaned, flags=re.IGNORECASE):
        return False
    return not re.match(
        r"^(?:Figure|Fig\.?)\s+\d+\s+(?:shows?|illustrates?|depicts?|describes?|summarizes?)\b",
        cleaned,
        flags=re.IGNORECASE,
    )


def _figure_caption_blocks(blocks: list[dict]) -> list[dict]:
    captions = []
    for block in blocks:
        if block.get("block_label") not in {"figure_title", "paragraph_title", "text"}:
            continue
        text = clean_block_text(block.get("block_content", ""))
        if is_formal_figure_legend(text):
            captions.append(block)
    return captions


def estimate_body_column_width(blocks: list[dict], page_width: int = 0) -> int:
    widths: list[int] = []
    for block in blocks:
        if block.get("block_label") not in {"text", "paragraph_title", "abstract"}:
            continue
        text = clean_block_text(block.get("block_content", ""))
        if (
            not text
            or is_subfigure_label(text)
            or re.match("^(?:Figure|Fig\\.?|Table)\\s+\\w+", text, flags=re.IGNORECASE)
        ):
            continue
        bbox = [int(value) for value in block.get("block_bbox", [0, 0, 0, 0])]
        width = _bbox_width(bbox)
        height = _bbox_height(bbox)
        if width <= 0 or height <= 0:
            continue
        if page_width and width >= int(page_width * 0.82):
            widths.append(width)
            continue
        if width >= 430:
            widths.append(width)
    if not widths:
        return int(page_width * 0.45) if page_width else 520
    widths.sort()
    return widths[len(widths) // 2]


def is_body_paragraph_like_text_block(
    block: dict,
    body_column_width: int = 0,
    cluster_bboxes: list[list[int]] | None = None,
    caption_bbox: list[int] | None = None,
) -> bool:
    if block.get("block_label") not in {"text", "paragraph_title", "abstract"}:
        return False
    text = clean_block_text(block.get("block_content", ""))
    if not text or is_formal_figure_legend(text) or is_subfigure_label(text):
        return False
    bbox = [int(value) for value in block.get("block_bbox", [0, 0, 0, 0])]
    width = _bbox_width(bbox)
    height = _bbox_height(bbox)
    if width <= 0 or height <= 0:
        return False
    if body_column_width and width < int(body_column_width * 0.72):
        return False
    if (not body_column_width) and width < 360:
        return False
    if height < 110:
        return False
    sentence_breaks = len(re.findall(r"[.!?](?:\s|$)", text))
    if len(text) < 180 and sentence_breaks < 2:
        return False
    if caption_bbox is not None:
        caption_gap = int(caption_bbox[1]) - int(bbox[3])
        if not (24 <= caption_gap <= 720):
            return False
    if cluster_bboxes:
        nearest_vertical_gap = None
        for cluster_bbox in cluster_bboxes:
            if _bbox_vertical_overlap(bbox, cluster_bbox) > 0:
                return False
            gap = None
            if int(bbox[3]) <= int(cluster_bbox[1]):
                gap = int(cluster_bbox[1]) - int(bbox[3])
            elif int(cluster_bbox[3]) <= int(bbox[1]):
                gap = int(bbox[1]) - int(cluster_bbox[3])
            if gap is None:
                continue
            if nearest_vertical_gap is None or gap < nearest_vertical_gap:
                nearest_vertical_gap = gap
        if nearest_vertical_gap is not None and nearest_vertical_gap > 140:
            return False
    return True


def _precaption_media_region(
    caption_bbox: list[int],
    cluster_bboxes: list[list[int]],
    blocks: list[dict] | None = None,
    body_column_width: int = 0,
) -> list[int] | None:
    relevant = [bbox for bbox in cluster_bboxes if int(bbox[3]) <= int(caption_bbox[1]) + 24]
    if len(relevant) < 1:
        return None
    if not blocks:
        return _union_bboxes(relevant)
    caption_y1 = int(caption_bbox[1])
    barrier_y = 0
    for block in blocks:
        label = block.get("block_label", "")
        if label not in {"text", "paragraph_title", "abstract"}:
            continue
        bbox = block.get("block_bbox", [0, 0, 0, 0])
        if int(bbox[3]) > caption_y1 + 24 or int(bbox[3]) <= barrier_y:
            continue
        text = clean_block_text(block.get("block_content", ""))
        if not text:
            continue
        if is_subfigure_label(text):
            continue
        if is_formal_figure_legend(text):
            barrier_y = max(barrier_y, int(bbox[3]))
            continue
        width = _bbox_width(bbox)
        height = _bbox_height(bbox)
        if width <= 0 or height <= 0:
            continue
        if body_column_width and width < int(body_column_width * 0.45):
            continue
        if not body_column_width and width < 320:
            continue
        if len(text) < 30:
            continue
        barrier_y = max(barrier_y, int(bbox[3]))
    for block in blocks:
        label = block.get("block_label", "")
        if label not in {"figure_title", "paragraph_title", "text"}:
            continue
        text = clean_block_text(block.get("block_content", ""))
        if not is_formal_figure_legend(text):
            continue
        bbox = block.get("block_bbox", [0, 0, 0, 0])
        if int(bbox[3]) <= caption_y1 and int(bbox[3]) > barrier_y:
            barrier_y = max(barrier_y, int(bbox[3]))
    filtered = [bbox for bbox in relevant if int(bbox[1]) >= barrier_y - 24]
    if len(filtered) < 1:
        return None
    return _union_bboxes(filtered)


def compute_precaption_composite_regions(blocks: list[dict], page_width: int = 0, page_height: int = 0) -> list[dict]:
    caption_blocks = _figure_caption_blocks(blocks)
    _, clusters = media_clusters(blocks)
    cluster_bboxes = [_cluster_bbox(cluster) for cluster in clusters]
    body_column_width = estimate_body_column_width(blocks, page_width=page_width)
    regions: list[dict] = []
    for caption in caption_blocks:
        caption_bbox = [int(value) for value in caption.get("block_bbox", [0, 0, 0, 0])]
        precaption_region = _precaption_media_region(
            caption_bbox, cluster_bboxes, blocks=blocks, body_column_width=body_column_width
        )
        if not precaption_region:
            continue
        region_blocks = []
        for block in blocks:
            label = block.get("block_label", "")
            bbox = [int(value) for value in block.get("block_bbox", [0, 0, 0, 0])]
            if bbox[3] > caption_bbox[1] + 24:
                continue
            if bbox[3] < precaption_region[1] - 240:
                continue
            vertical_overlap_with_region = _bbox_vertical_overlap(bbox, precaption_region)
            near_region_side = vertical_overlap_with_region > 0 and (
                0 <= precaption_region[0] - bbox[2] <= 80 or 0 <= bbox[0] - precaption_region[2] <= 80
            )
            intersects_region = (
                _bbox_horizontal_overlap(bbox, precaption_region) > 0
                or precaption_region[0] - 24 <= _bbox_center_x(bbox) <= precaption_region[2] + 24
                or near_region_side
            )
            if not intersects_region:
                continue
            if label in {"image", "chart"}:
                if _bbox_vertical_overlap(bbox, precaption_region) <= 0 and bbox[3] < precaption_region[1] - 80:
                    continue
                region_blocks.append(block)
                continue
            if label in {"text", "paragraph_title"}:
                if bbox[3] < precaption_region[1] - 80:
                    continue
                width = _bbox_width(bbox)
                text = clean_block_text(block.get("block_content", ""))
                if is_body_paragraph_like_text_block(
                    block,
                    body_column_width=body_column_width,
                    cluster_bboxes=cluster_bboxes,
                    caption_bbox=caption_bbox,
                ):
                    continue
                if (
                    text
                    and (
                        not re.match(
                            "^(?:Extended\\s+Data\\s+Fig\\.?|Extended\\s+Data\\s+Figure|Figure|Fig\\.?|Table)\\s+\\w+",
                            text,
                            flags=re.IGNORECASE,
                        )
                    )
                    and (
                        width <= int(max(body_column_width, 1) * 0.78)
                        or is_embedded_figure_text_block(block, blocks, page_width=page_width, page_height=page_height)
                    )
                ):
                    region_blocks.append(block)
        media_ids = {block.get("block_id") for block in region_blocks if block.get("block_label") in {"image", "chart"}}
        text_ids = {
            block.get("block_id") for block in region_blocks if block.get("block_label") in {"text", "paragraph_title"}
        }
        if len(media_ids) < 1 or not text_ids or len(region_blocks) < 3:
            continue
        region_bbox = [
            min(int(block["block_bbox"][0]) for block in region_blocks),
            min(int(block["block_bbox"][1]) for block in region_blocks),
            max(int(block["block_bbox"][2]) for block in region_blocks),
            max(int(block["block_bbox"][3]) for block in region_blocks),
        ]
        regions.append(
            {
                "bbox": region_bbox,
                "block_ids": {block.get("block_id") for block in region_blocks},
                "caption_block_id": caption.get("block_id"),
            }
        )
    return regions


def is_embedded_figure_text_block(block: dict, blocks: list[dict], page_width: int = 0, page_height: int = 0) -> bool:
    label = block.get("block_label", "")
    if label not in {"text", "paragraph_title"}:
        return False
    if label == "paragraph_title":
        return False
    text = clean_block_text(block.get("block_content", ""))
    if not text:
        return False
    if is_formal_figure_legend(text):
        return False
    bbox = [int(value) for value in block.get("block_bbox", [0, 0, 0, 0])]
    width = _bbox_width(bbox)
    height = _bbox_height(bbox)
    if width <= 0 or height <= 0:
        return False
    if label == "paragraph_title" and is_subfigure_label(text):
        return True
    caption_blocks = _figure_caption_blocks(blocks)
    cluster_index, clusters = media_clusters(blocks)
    cluster_bboxes = [_cluster_bbox(cluster) for cluster in clusters]
    body_column_width = estimate_body_column_width(blocks, page_width=page_width)
    del cluster_index
    nearest_media = None
    nearest_media_distance = None
    close_media_count = 0
    stacked_media_above = False
    stacked_media_below = False
    side_media = False
    for cluster_bbox in cluster_bboxes:
        horizontal_ratio = _bbox_horizontal_overlap_ratio(bbox, cluster_bbox)
        vertical_overlap = _bbox_vertical_overlap(bbox, cluster_bbox)
        top_gap = int(bbox[1]) - int(cluster_bbox[3])
        bottom_gap = int(cluster_bbox[1]) - int(bbox[3])
        center_inside_x = int(cluster_bbox[0]) <= _bbox_center_x(bbox) <= int(cluster_bbox[2])
        center_inside_y = int(cluster_bbox[1]) <= _bbox_center_y(bbox) <= int(cluster_bbox[3])
        if (
            horizontal_ratio >= 0.45
            and (0 <= top_gap <= 48 or 0 <= bottom_gap <= 48 or vertical_overlap > 0)
            or (center_inside_x and abs(_bbox_center_y(bbox) - _bbox_center_y(cluster_bbox)) <= max(80, height * 4))
        ):
            close_media_count += 1
        if horizontal_ratio >= 0.5 and 0 <= top_gap <= 90:
            stacked_media_above = True
        if horizontal_ratio >= 0.5 and 0 <= bottom_gap <= 90:
            stacked_media_below = True
        if vertical_overlap > 0 and (
            0 <= int(bbox[0]) - int(cluster_bbox[2]) <= 60 or 0 <= int(cluster_bbox[0]) - int(bbox[2]) <= 60
        ):
            side_media = True
        dx = 0
        if int(bbox[2]) < int(cluster_bbox[0]):
            dx = int(cluster_bbox[0]) - int(bbox[2])
        elif int(cluster_bbox[2]) < int(bbox[0]):
            dx = int(bbox[0]) - int(cluster_bbox[2])
        dy = 0
        if int(bbox[3]) < int(cluster_bbox[1]):
            dy = int(cluster_bbox[1]) - int(bbox[3])
        elif int(cluster_bbox[3]) < int(bbox[1]):
            dy = int(bbox[1]) - int(cluster_bbox[3])
        distance = dx + dy
        if nearest_media_distance is None or distance < nearest_media_distance:
            nearest_media_distance = distance
            nearest_media = cluster_bbox
            del center_inside_y
    nearest_caption_above = None
    nearest_caption_below = None
    for caption in caption_blocks:
        cb = [int(value) for value in caption.get("block_bbox", [0, 0, 0, 0])]
        if cb[3] <= bbox[1] and (nearest_caption_above is None or cb[3] > nearest_caption_above[3]):
            nearest_caption_above = cb
        if cb[1] >= bbox[3] and (nearest_caption_below is None or cb[1] < nearest_caption_below[1]):
            nearest_caption_below = cb
    media_between_block_and_caption = 0
    media_x_covering_block = 0
    precaption_region = None
    if nearest_caption_below:
        precaption_region = _precaption_media_region(nearest_caption_below, cluster_bboxes)
        for cluster_bbox in cluster_bboxes:
            if cluster_bbox[1] >= nearest_caption_below[1] + 24:
                continue
            if cluster_bbox[3] <= bbox[1] - 24:
                continue
            media_between_block_and_caption += 1
            if (
                cluster_bbox[0] - 24 <= _bbox_center_x(bbox) <= cluster_bbox[2] + 24
                or _bbox_horizontal_overlap_ratio(bbox, cluster_bbox) >= 0.35
            ):
                media_x_covering_block += 1
    if nearest_caption_above:
        caption_gap = bbox[1] - nearest_caption_above[3]
        left_align_gap = abs(bbox[0] - nearest_caption_above[0])
        effective_page_width = max(page_width, width)
        if (
            0 <= caption_gap <= 72
            and left_align_gap <= 80
            and (width >= int(effective_page_width * 0.45))
            and (not stacked_media_above)
        ):
            return False
    if is_body_paragraph_like_text_block(
        block,
        body_column_width=body_column_width,
        cluster_bboxes=cluster_bboxes,
        caption_bbox=nearest_caption_below,
    ):
        return False
    score = 0.0
    if is_subfigure_label(text):
        score += 4.0
    if width <= int(max(body_column_width, 1) * 0.78):
        score += 1.4
    elif width <= int(max(body_column_width, 1) * 0.9):
        score += 0.5
    if label == "paragraph_title" and len(text) <= 24:
        score += 0.8
    if height <= 26:
        score += 0.8
    elif height <= 34:
        score += 0.4
    if page_width and width <= int(page_width * 0.22):
        score += 0.5
    if close_media_count >= 2:
        score += 2.0
    elif close_media_count == 1:
        score += 1.1
    if stacked_media_above and stacked_media_below:
        score += 2.2
    elif stacked_media_above or stacked_media_below:
        score += 0.9
    if side_media:
        score += 1.0
    if nearest_caption_below and nearest_media:
        caption_gap = nearest_caption_below[1] - bbox[3]
        media_gap = min(
            abs(bbox[1] - nearest_media[3]),
            abs(nearest_media[1] - bbox[3]),
            abs(_bbox_center_y(bbox) - _bbox_center_y(nearest_media)),
        )
        if 0 <= caption_gap <= 120 and media_gap <= 80:
            score += 0.8
    if nearest_caption_below:
        caption_gap = nearest_caption_below[1] - bbox[3]
        if 0 <= caption_gap <= 520 and media_between_block_and_caption >= 3:
            score += 0.9
        if 0 <= caption_gap <= 520 and media_x_covering_block >= 2:
            score += 1.3
        if (
            0 <= caption_gap <= 520
            and media_x_covering_block >= 1
            and (width <= int(max(page_width, width) * 0.82))
            and (height <= 96)
        ):
            score += 0.8
        if precaption_region:
            region_overlap = _bbox_horizontal_overlap_ratio(bbox, precaption_region)
            within_region_y = (
                int(precaption_region[1]) - 24 <= int(bbox[1]) <= int(precaption_region[3]) + 24
                and int(bbox[3]) <= int(nearest_caption_below[1]) + 24
            )
            if region_overlap >= 0.55 and within_region_y:
                score += 1.5
            if (
                int(precaption_region[0]) - 24 <= _bbox_center_x(bbox) <= int(precaption_region[2]) + 24
                and within_region_y
                and (media_between_block_and_caption >= 2)
            ):
                score += 1.1
    if len(text) <= 22:
        score += 0.15
    return score >= 2.6


def is_embedded_vision_footnote_block(
    block: dict, blocks: list[dict], page_width: int = 0, page_height: int = 0
) -> bool:
    if block.get("block_label") != "vision_footnote":
        return False
    if is_formal_figure_legend(block.get("block_content", "")):
        return False
    bbox = [int(value) for value in block.get("block_bbox", [0, 0, 0, 0])]
    if _bbox_width(bbox) <= 0 or _bbox_height(bbox) <= 0:
        return False
    composite_regions = compute_precaption_composite_regions(blocks, page_width=page_width, page_height=page_height)
    for region in composite_regions:
        region_bbox = region.get("bbox", [0, 0, 0, 0])
        if _bbox_contains(tuple(region_bbox), tuple(bbox), margin=36):
            return True
    _, clusters = media_clusters(blocks)
    for cluster in clusters:
        cluster_bbox = _cluster_bbox(cluster)
        if _bbox_contains(tuple(cluster_bbox), tuple(bbox), margin=24):
            return True
        if _bbox_horizontal_overlap(bbox, cluster_bbox) > 0 and _bbox_vertical_overlap(bbox, cluster_bbox) > 0:
            return True
        vertical_overlap = _bbox_vertical_overlap(bbox, cluster_bbox)
        near_side = (
            vertical_overlap > 0
            and _bbox_width(bbox) <= 180
            and (0 <= int(cluster_bbox[0]) - int(bbox[2]) <= 60 or 0 <= int(bbox[0]) - int(cluster_bbox[2]) <= 60)
        )
        if near_side:
            return True
    return False


def caption_group_assignments(blocks: list[dict]) -> tuple[dict[int, list[dict]], dict[int, list[dict]]]:
    figure_captions = []
    table_captions = []
    for block in blocks:
        if block.get("block_label") not in {"figure_title", "paragraph_title", "text"}:
            continue
        text = clean_block_text(block.get("block_content", ""))
        if is_numbered_figure_caption(text):
            figure_captions.append(block)
        elif re.match("^(?:Table|Extended\\s+Data\\s+Table|Supplementary\\s+Table)\\s+\\d+", text, flags=re.IGNORECASE):
            table_captions.append(block)
    figure_map: dict[int, list[dict]] = {}
    table_map: dict[int, list[dict]] = {}
    for block in blocks:
        label = block.get("block_label")
        bbox = block.get("block_bbox", [0, 0, 0, 0])
        if label in {"image", "chart"}:
            best_caption = None
            best_distance = None
            for caption in figure_captions:
                cb = caption.get("block_bbox", [0, 0, 0, 0])
                if bbox[1] < cb[1]:
                    horizontal_overlap = _bbox_horizontal_overlap(bbox, cb)
                    center_delta = abs(_bbox_center_x(bbox) - _bbox_center_x(cb))
                    width_gate = max(140, min(_bbox_width(bbox), _bbox_width(cb)) * 0.6)
                    if horizontal_overlap <= 0 and center_delta > width_gate:
                        continue
                    distance = cb[1] - bbox[1]
                    if best_distance is None or distance < best_distance:
                        best_caption = caption
                        best_distance = distance
            if best_caption:
                figure_map.setdefault(best_caption["block_id"], []).append(block)
        elif label == "table":
            best_caption = None
            best_distance = None
            for caption in table_captions:
                cb = caption.get("block_bbox", [0, 0, 0, 0])
                distance = min(abs(bbox[1] - cb[3]), abs(cb[1] - bbox[3]))
                if distance < 260 and (best_distance is None or distance < best_distance):
                    best_caption = caption
                    best_distance = distance
            if best_caption:
                table_map.setdefault(best_caption["block_id"], []).append(block)
    return (figure_map, table_map)


def _apply_layered_body_reorder(blocks: list[dict], page_width: int, page_height: int) -> list[dict]:
    """Column-major sort for two-column pages.

    Groups blocks by column (left/center first, right second), each
    sorted by y-position.  Single-column pages pass through unchanged.
    """
    if not page_width or len(blocks) < 4:
        return blocks

    mid = page_width / 2

    def _col(b):
        bb = b.get("block_bbox") or b.get("bbox")
        if not bb or len(bb) < 4:
            return 0
        xc = (bb[0] + bb[2]) / 2
        w = bb[2] - bb[0]
        if (bb[0] < mid and bb[2] > mid) or w >= page_width * 0.55:
            return 0
        return 1 if xc >= mid else 0

    has_left = any(_col(b) == 0 for b in blocks)
    has_right = any(_col(b) == 1 for b in blocks)
    if not (has_left and has_right):
        return blocks

    left = sorted(
        [b for b in blocks if _col(b) == 0],
        key=lambda b: (b.get("block_bbox") or b.get("bbox") or [0, 0, 0, 0])[1],
    )
    right = sorted(
        [b for b in blocks if _col(b) == 1],
        key=lambda b: (b.get("block_bbox") or b.get("bbox") or [0, 0, 0, 0])[1],
    )
    return left + right


def render_page_blocks(
    vault: Path, page_index: int, result: dict, images_dir: Path, page_cache_dir: Path, pdf_doc=None
) -> list[str]:
    pruned = result.get("prunedResult", {})
    ocr_width = int(pruned.get("width", 0) or 0)
    blocks = sorted(pruned.get("parsing_res_list", []), key=block_sort_key)
    ocr_height = int(pruned.get("height", 0) or 0)
    if page_index != 1:
        blocks = validate_block_order(blocks, ocr_width)
        blocks = _apply_layered_body_reorder(blocks, ocr_width, ocr_height)
    raw_reference_blocks = [block for block in blocks if block.get("block_label") == "reference_content"]
    first_reference_y = min(
        (block.get("block_bbox", [0, 10**9, 0, 0])[1] for block in raw_reference_blocks), default=10**9
    )
    page_image = render_pdf_page_cached(
        pdf_doc, page_index, ocr_width, ocr_height, page_cache_dir / f"page_{page_index:03d}.png"
    )
    if not page_image:
        page_image = ensure_page_image_cached(
            result.get("inputImage", ""), page_cache_dir / f"page_{page_index:03d}.jpg"
        )
    cluster_index, clusters = media_clusters(blocks)
    figure_caption_map, table_caption_map = caption_group_assignments(blocks)
    composite_regions = compute_precaption_composite_regions(blocks, page_width=ocr_width, page_height=ocr_height)
    composite_by_block_id: dict[int, dict] = {}
    for region in composite_regions:
        region["rendered"] = False
        for block_id in region.get("block_ids", set()):
            composite_by_block_id[block_id] = region
    caption_linked_media_ids = {
        item.get("block_id")
        for media_list in list(figure_caption_map.values()) + list(table_caption_map.values())
        for item in media_list
    }
    rendered: list[str] = [f"<!-- page {page_index} -->"]
    reference_blocks: list[dict] = []
    reference_continuations: list[dict] = []
    footnotes: list[str] = []
    footnote_counter = 0
    references_heading_seen = False
    references_section_active = False
    references_emitted = False
    references_insert_index: int | None = None
    rendered_cluster_ids: set[int] = set()
    rendered_caption_media_ids: set[int] = set()
    first_page_meta_done = page_index != 1
    affiliation_buffer: list[str] = []
    deferred_meta: list[str] = []
    for block in blocks:
        label = block.get("block_label", "")
        content = block.get("block_content", "")
        bbox = block.get("block_bbox", [0, 0, 0, 0])
        composite_region = composite_by_block_id.get(block.get("block_id"))
        if composite_region:
            if not composite_region.get("rendered"):
                composite_region["rendered"] = True
                region_bbox = composite_region["bbox"]
                asset_path = (
                    images_dir
                    / "blocks"
                    / f"page_{page_index:03d}_figure_{region_bbox[0]}_{region_bbox[1]}_{region_bbox[2]}_{region_bbox[3]}.jpg"
                )
                if page_image and crop_block_asset(page_image, region_bbox, asset_path):
                    rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault)))
            continue
        if label in {"header", "header_image", "footer", "footer_image", "number"}:
            continue
        if label == "doc_title":
            rendered.append(f"# {clean_block_text(content)}")
            continue
        if label == "paragraph_title":
            title = clean_block_text(content)
            if (
                title.lower() == "check for updates"
                or is_subfigure_label(title)
                or is_reference_tail_noise_line(title)
                or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height)
            ):
                continue
            if page_index == 1 and affiliation_buffer:
                rendered.extend(affiliation_buffer)
                affiliation_buffer.clear()
            if page_index == 1 and deferred_meta:
                rendered.extend(deferred_meta)
                deferred_meta.clear()
                first_page_meta_done = True
            if references_heading_seen and (not references_emitted) and title.lower() != "references":
                if reference_blocks:
                    append_reference_section(rendered, reference_blocks, reference_continuations)
                    references_emitted = True
                references_section_active = False
            if title.lower() == "abstract" or re.match(r"^\d+\s", title):
                rendered.append(f"## {title}")
            else:
                rendered.append(f"### {title}")
            if title.lower() == "references":
                references_heading_seen = True
                references_section_active = True
                references_insert_index = len(rendered)
            continue
        if label == "abstract":
            if page_index == 1 and affiliation_buffer:
                rendered.extend(affiliation_buffer)
                affiliation_buffer.clear()
            if page_index == 1 and deferred_meta:
                rendered.extend(deferred_meta)
                deferred_meta.clear()
                first_page_meta_done = True
            rendered.append(clean_block_text(content))
            continue
        if label == "reference_content":
            text = clean_block_text(content)
            if text and (not is_reference_tail_noise_line(text)):
                if is_reference_lead_block(text):
                    reference_blocks.append(block)
                else:
                    reference_continuations.append(block)
            continue
        if label == "text":
            text = clean_block_text(content)
            if (
                not text
                or is_subfigure_label(text)
                or is_frontmatter_noise_line(text)
                or is_reference_tail_noise_line(text)
                or is_embedded_figure_text_block(block, blocks, page_width=ocr_width, page_height=ocr_height)
            ):
                continue
            if references_section_active and raw_reference_blocks and bbox[1] >= first_reference_y - 10:
                reference_continuations.append(block)
                continue
            if is_numbered_figure_caption(text):
                linked_media = figure_caption_map.get(block.get("block_id"), []) or table_caption_map.get(
                    block.get("block_id"), []
                )
                if linked_media and page_image:
                    rendered_caption_media_ids.update(item.get("block_id") for item in linked_media)
                    union_bbox = [
                        min(item["block_bbox"][0] for item in linked_media),
                        min(item["block_bbox"][1] for item in linked_media),
                        max(item["block_bbox"][2] for item in linked_media),
                        max(item["block_bbox"][3] for item in linked_media),
                    ]
                    asset_kind = "figure" if block.get("block_id") in figure_caption_map else "table"
                    asset_path = (
                        images_dir
                        / "blocks"
                        / f"page_{page_index:03d}_{asset_kind}_{union_bbox[0]}_{union_bbox[1]}_{union_bbox[2]}_{union_bbox[3]}.jpg"
                    )
                    if crop_block_asset(page_image, union_bbox, asset_path):
                        rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault)))
                    if asset_kind == "table":
                        for item in linked_media:
                            if item.get("block_label") == "table" and item.get("block_content"):
                                rendered.append(clean_block_text(item.get("block_content", "")))
                                break
                rendered.append(text)
                continue
            if page_index == 1 and (not first_page_meta_done):
                if handle_first_page_metadata_lines(text, rendered, affiliation_buffer, deferred_meta, footnotes):
                    continue
            rendered.append(text)
            continue
        if label == "display_formula":
            formula = clean_block_text(content)
            formula = formula.strip()
            if formula.startswith("$$") and formula.endswith("$$") and (len(formula) >= 4):
                formula = formula[2:-2].strip()
            rendered.append(f"$$\n{formula}\n$$")
            continue
        if label == "formula_number":
            rendered.append(clean_block_text(content))
            continue
        if label in {"table", "image", "chart"}:
            if block.get("block_id") in caption_linked_media_ids:
                continue
            if block.get("block_id") in rendered_caption_media_ids:
                continue
            if label in {"image", "chart"}:
                cluster_id = cluster_index.get(block.get("block_id", -1))
                if cluster_id is None or cluster_id in rendered_cluster_ids:
                    continue
                rendered_cluster_ids.add(cluster_id)
                cluster_blocks = clusters[cluster_id]
                bbox = [
                    min(item["block_bbox"][0] for item in cluster_blocks),
                    min(item["block_bbox"][1] for item in cluster_blocks),
                    max(item["block_bbox"][2] for item in cluster_blocks),
                    max(item["block_bbox"][3] for item in cluster_blocks),
                ]
                asset_name = f"page_{page_index:03d}_figure_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg"
            else:
                asset_name = f"page_{page_index:03d}_{label}_{bbox[0]}_{bbox[1]}_{bbox[2]}_{bbox[3]}.jpg"
            asset_path = images_dir / "blocks" / asset_name
            if page_image and crop_block_asset(page_image, bbox, asset_path):
                rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault)))
            if label == "table" and content:
                rendered.append(clean_block_text(content))
            continue
        if label == "figure_title":
            caption_text = clean_block_text(content)
            if is_subfigure_label(caption_text):
                continue
            if not is_formal_figure_legend(caption_text):
                continue
            linked_media = figure_caption_map.get(block.get("block_id"), []) or table_caption_map.get(
                block.get("block_id"), []
            )
            if linked_media and page_image:
                rendered_caption_media_ids.update(item.get("block_id") for item in linked_media)
                union_bbox = [
                    min(item["block_bbox"][0] for item in linked_media),
                    min(item["block_bbox"][1] for item in linked_media),
                    max(item["block_bbox"][2] for item in linked_media),
                    max(item["block_bbox"][3] for item in linked_media),
                ]
                asset_kind = "figure" if block.get("block_id") in figure_caption_map else "table"
                asset_path = (
                    images_dir
                    / "blocks"
                    / f"page_{page_index:03d}_{asset_kind}_{union_bbox[0]}_{union_bbox[1]}_{union_bbox[2]}_{union_bbox[3]}.jpg"
                )
                if crop_block_asset(page_image, union_bbox, asset_path):
                    rendered.append(_image_embed_for_obsidian(asset_path.relative_to(vault)))
                if asset_kind == "table":
                    for item in linked_media:
                        if item.get("block_label") == "table" and item.get("block_content"):
                            rendered.append(clean_block_text(item.get("block_content", "")))
                            break
            rendered.append(caption_text)
            continue
        if label == "vision_footnote":
            if is_formal_figure_legend(content):
                rendered.append(clean_block_text(content))
                continue
            if is_embedded_vision_footnote_block(block, blocks, page_width=ocr_width, page_height=ocr_height):
                continue
            entries = parse_vision_footnote_entries(content)
            marker_to_id = {}
            for marker, body in entries:
                footnote_counter += 1
                footnote_id = f"p{page_index}-fn{footnote_counter}"
                marker_to_id[marker] = footnote_id
                footnotes.append(f"[^{footnote_id}]: {body or clean_block_text(content)}")
            if rendered and marker_to_id:
                last = rendered[-1]
                if last.startswith("<table>"):
                    rendered[-1] = attach_table_footnotes(last, marker_to_id)
                else:
                    for marker, footnote_id in marker_to_id.items():
                        rendered[-1] = attach_footnote_reference(rendered[-1], marker, footnote_id)
            continue
    if footnotes:
        rendered.append("")
        rendered.extend(footnotes)
    rendered = dedupe_page_media_lines(rendered)
    if reference_blocks and not references_emitted:
        reference_lines = build_reference_section_lines(reference_blocks, reference_continuations)
        if references_insert_index is not None:
            rendered[references_insert_index:references_insert_index] = reference_lines
        else:
            rendered.extend(reference_lines)
    return [part for part in rendered if part]


def postprocess_ocr_result(vault: Path, key: str, all_results: list[dict]) -> tuple[int, str, str, str]:
    paths = pipeline_paths(vault)
    ocr_root = paths["ocr"] / key
    json_dir = ocr_root / "json"
    images_dir = ocr_root / "images"
    page_cache_dir = ocr_root / "pages"
    meta_path = ocr_root / "meta.json"
    json_dir.mkdir(parents=True, exist_ok=True)
    images_dir.mkdir(parents=True, exist_ok=True)
    page_cache_dir.mkdir(parents=True, exist_ok=True)
    page_num = 0
    meta = read_json(meta_path) if meta_path.exists() else {}
    source_pdf = Path(meta.get("source_pdf", "")) if meta.get("source_pdf") else None
    pdf_doc = None
    try:
        if source_pdf and source_pdf.exists():
            pdf_doc = fitz.open(str(source_pdf))
        for page_payload in all_results:
            for res in page_payload.get("layoutParsingResults", []):
                page_num += 1
                render_page_blocks(vault, page_num, res, images_dir, page_cache_dir, pdf_doc=pdf_doc)
    finally:
        if pdf_doc is not None:
            pdf_doc.close()
    write_json(json_dir / "result.json", all_results)

    # --- Phase 1: raw metadata and source metadata ---
    artifacts = artifact_paths_for_key(vault, key)

    # raw_meta.json
    source_pdf_path = Path(meta.get("source_pdf", "")) if meta.get("source_pdf") else None
    pdf_fingerprint = compute_pdf_fingerprint(source_pdf_path) if source_pdf_path and source_pdf_path.exists() else "unknown"
    result_hash = compute_json_hash(all_results)
    now_iso = datetime.now(timezone.utc).isoformat()
    raw_meta = {
        "ocr_provider": meta.get("ocr_provider", "PaddleOCR"),
        "ocr_model": meta.get("ocr_model", "PaddleOCR"),
        "ocr_raw_schema_version": "1.0.0",
        "created_at": now_iso,
        "updated_at": now_iso,
        "pdf_fingerprint": pdf_fingerprint,
        "result_hash": result_hash,
    }
    write_json(artifacts.raw_meta, raw_meta)

    # source_metadata.json
    source_meta = {
        "zotero_key": meta.get("zotero_key", key),
        "source_pdf": meta.get("source_pdf", ""),
        "title": meta.get("title", ""),
        "authors": meta.get("authors", []),
        "year": meta.get("year", 0),
        "journal": meta.get("journal", ""),
        "doi": meta.get("doi", ""),
        "bbt_citekey": meta.get("bbt_citekey", ""),
        "source": "zotero_bbt",
    }
    write_json(artifacts.source_metadata, source_meta)

    # Emit canonical raw blocks
    artifacts.blocks_raw.parent.mkdir(parents=True, exist_ok=True)
    artifacts.blocks_structured.parent.mkdir(parents=True, exist_ok=True)
    all_raw_blocks = build_raw_blocks_for_result_lines(key, all_results)
    write_raw_blocks_jsonl(artifacts.blocks_raw, all_raw_blocks)

    # Backfill span_metadata from source PDF (not from OCR engine)
    source_pdf_path = Path(meta.get("source_pdf", "")) if meta.get("source_pdf") else None
    if source_pdf_path and source_pdf_path.exists():
        from paperforge.worker.ocr_pdf_spans import backfill_span_metadata_from_pdf
        backfill_span_metadata_from_pdf(all_raw_blocks, source_pdf_path)
        write_raw_blocks_jsonl(artifacts.blocks_raw, all_raw_blocks)

    structured, doc_structure = build_structured_blocks(
        all_raw_blocks,
        structure_output_dir=artifacts.blocks_structured.parent,
    )
    write_structured_blocks_jsonl(artifacts.blocks_structured, structured)
    # Write role-level span profiles
    from paperforge.worker.ocr_profiles import write_role_span_profiles
    write_role_span_profiles(structured, artifacts.blocks_structured.parent)

    # --- Phase 2: metadata resolution ---
    metadata_dir = ocr_root / "metadata"
    metadata_dir.mkdir(parents=True, exist_ok=True)
    frontmatter_candidates = extract_frontmatter_candidates(artifacts.blocks_structured)
    resolved = resolve_metadata(source_meta, frontmatter_candidates)
    write_resolved_metadata(metadata_dir / "resolved_metadata.json", resolved)

    # --- Phase 2: figure inventory ---
    figure_inventory = build_figure_inventory(structured)
    write_figure_inventory(
        artifacts.blocks_structured.parent / "figure_inventory.json",
        figure_inventory,
    )

    # --- Phase 2: table inventory ---
    table_inventory = build_table_inventory(structured)
    write_table_inventory(
        artifacts.blocks_structured.parent / "table_inventory.json",
        table_inventory,
    )

    # --- Phase 2: object artifacts ---
    from paperforge.worker.ocr_objects import extract_and_write_objects

    ocr_asset_root = ocr_root / "assets"
    ocr_render_root = ocr_root / "render"

    page_dimensions_by_page: dict[int, tuple[int, int]] = {}
    for block in structured:
        page = int(block.get("page", 0) or 0)
        width = int(block.get("page_width", 0) or 0)
        height = int(block.get("page_height", 0) or 0)
        if page and width and height and page not in page_dimensions_by_page:
            page_dimensions_by_page[page] = (width, height)

    extract_and_write_objects(
        pdf_path=source_pdf_path,
        figure_inventory=figure_inventory,
        table_inventory=table_inventory,
        asset_root=ocr_asset_root,
        render_root=ocr_render_root,
        page_dimensions_by_page=page_dimensions_by_page,
    )

    # --- Phase 3: structured renderer ---
    from paperforge.worker.ocr_render import render_fulltext_markdown, write_render_outputs

    markdown = render_fulltext_markdown(
        structured_blocks=structured,
        resolved_metadata=resolved,
        figure_inventory=figure_inventory,
        table_inventory=table_inventory,
        page_count=page_num,
        document_structure=doc_structure,
    )
    write_render_outputs(
        render_root=ocr_root / "render",
        compat_fulltext=ocr_root / "fulltext.md",
        markdown=markdown,
    )

    # --- Phase 3: OCR health report ---
    from paperforge.worker.ocr_health import build_ocr_health, write_ocr_health

    health_report = build_ocr_health(
        page_count=page_num,
        raw_blocks_count=len(all_raw_blocks),
        structured_blocks=structured,
        figure_inventory=figure_inventory,
        table_inventory=table_inventory,
    )
    write_ocr_health(ocr_root / "health", health_report)
    meta["ocr_health_overall"] = health_report["overall"]

    # --- Phase 5: role-based OCR index ---
    from paperforge.worker.ocr_index import build_role_indexes, write_role_index

    role_indexes = build_role_indexes(
        structured_blocks=structured,
        resolved_metadata=resolved,
    )
    write_role_index(ocr_root / "index", role_indexes)

    # Update meta.json with version payloads
    ocr_model = meta.get("ocr_model", meta.get("ocr_provider", "PaddleOCR"))
    version_payload = build_version_payload(
        pdf_fingerprint=pdf_fingerprint,
        result_json_hash=result_hash,
        ocr_model=ocr_model,
    )
    meta["raw_version"] = version_payload["raw_version"]
    meta["derived_version"] = version_payload["derived_version"]

    from paperforge.worker.ocr_versions import classify_version_state, expected_derived_payload, expected_raw_payload

    state = classify_version_state(
        meta=meta,
        expected_raw=expected_raw_payload(ocr_model=meta.get("raw_version", {}).get("ocr_model", "unknown")),
        expected_derived=expected_derived_payload(),
    )
    meta["raw_upgradable"] = state["raw_upgradable"]
    meta["derived_stale"] = state["derived_stale"]
    meta["version_state_updated_at"] = __import__("datetime").datetime.now().isoformat()
    try:
        meta["legacy_images_path"] = str(images_dir.relative_to(vault)).replace("\\", "/")
    except ValueError:
        meta["legacy_images_path"] = str(images_dir)
    meta["path_map"] = {"structured_truth": "assets/", "legacy_compat": "images/"}
    write_json(meta_path, meta)

    fulltext_path = ocr_root / "fulltext.md"
    markdown_dir = ocr_root / "markdown"
    if markdown_dir.exists():
        shutil.rmtree(markdown_dir)
    markdown_path = str(fulltext_path.relative_to(paths["vault"])).replace("\\", "/") if page_num else ""
    json_path = str((json_dir / "result.json").relative_to(paths["vault"])).replace("\\", "/")
    fulltext_md_path = str(fulltext_path.resolve())
    return (page_num, markdown_path, json_path, fulltext_md_path)


def _workspace_fulltext_candidates(lit_root: Path, zotero_key: str) -> list[Path]:
    if not lit_root.exists():
        return []
    matches: list[Path] = []
    for candidate in lit_root.rglob(f"{zotero_key} - *"):
        if candidate.is_dir():
            fulltext_path = candidate / "fulltext.md"
            if fulltext_path.exists():
                matches.append(fulltext_path)
    return matches


def _rewrite_note_fields(text: str, **fields: object) -> str:
    updated = text
    for field, value in fields.items():
        updated = _sync.update_frontmatter_field(updated, field, value)
    return updated


def ocr_redo_papers(vault: Path, dry_run: bool = False, verbose: bool = False, no_progress: bool = False) -> int:
    """Scan for papers with ocr_redo: true, reset and immediately rerun OCR.

    Spec 2.6: paperforge ocr redo [--dry-run]
    """
    from paperforge.adapters.obsidian_frontmatter import (
        extract_preserved_ocr_redo,
    )

    paths = pipeline_paths(vault)
    ocr_root = paths.get("ocr")
    lit_root = paths.get("literature")

    if not lit_root or not lit_root.exists():
        if verbose:
            logger.info("No literature directory found, nothing to redo")
        return 0

    redo_entry_map: dict[str, tuple[Path, str]] = {}
    for note_file in sorted(lit_root.rglob("*.md")):
        if note_file.name in ("fulltext.md", "deep-reading.md", "discussion.md"):
            continue
        try:
            text = note_file.read_text(encoding="utf-8")
        except Exception:
            continue
        if not extract_preserved_ocr_redo(text):
            continue
        key_match = re.search(r"^zotero_key:\s*(.+)$", text, re.MULTILINE)
        if not key_match:
            continue
        zotero_key = key_match.group(1).strip().strip('"').strip("'")
        if not zotero_key or not re.match(r"^[A-Za-z0-9]{8}$", zotero_key):
            if verbose:
                logger.warning("Skipping invalid zotero_key: %s", zotero_key)
            continue
        existing = redo_entry_map.get(zotero_key)
        if existing is None or note_file.name == f"{zotero_key}.md":
            redo_entry_map[zotero_key] = (note_file, text)

    redo_entries = [(key, note_file, text) for key, (note_file, text) in sorted(redo_entry_map.items())]

    if not redo_entries:
        print("No papers with ocr_redo: true found", flush=True)
        return 0

    print(f"Found {len(redo_entries)} paper(s) with ocr_redo: true:", flush=True)
    for zotero_key, note_file, _text in redo_entries:
        print(f"  {zotero_key}  ({note_file.parent.name})", flush=True)

    if dry_run:
        print("\nDry-run mode: no changes made. Run without --dry-run to rerun OCR.", flush=True)
        return 0

    reset_keys = []
    for zotero_key, note_file, text in redo_entries:
        for workspace_fulltext in _workspace_fulltext_candidates(lit_root, zotero_key):
            workspace_fulltext.unlink(missing_ok=True)
            if verbose:
                logger.info("Deleted workspace fulltext for %s", zotero_key)
        ocr_dir = ocr_root / zotero_key if ocr_root else None
        if ocr_dir and ocr_dir.exists():
            shutil.rmtree(ocr_dir)
            if verbose:
                logger.info("Deleted OCR directory for %s", zotero_key)

        text = _rewrite_note_fields(
            text,
            do_ocr=True,
            ocr_status="pending",
            fulltext_md_path="",
            ocr_redo=True,
        )
        note_file.write_text(text, encoding="utf-8")
        reset_keys.append(zotero_key)

    print(f"Reset {len(reset_keys)} paper(s): {', '.join(reset_keys)}", flush=True)
    print("Launching OCR immediately for reset papers...", flush=True)

    exit_code = run_ocr(vault, verbose=verbose, no_progress=no_progress, selected_keys=set(reset_keys))

    success_keys: list[str] = []
    failed_keys: list[str] = []
    for zotero_key, note_file, _original_text in redo_entries:
        current_text = note_file.read_text(encoding="utf-8")
        meta_path = ocr_root / zotero_key / "meta.json" if ocr_root else None
        meta = read_json(meta_path) if meta_path and meta_path.exists() else {}
        status, _error = validate_ocr_meta(paths, meta) if meta else ("pending", "")
        current_text = _rewrite_note_fields(current_text, ocr_status=status)
        if status == "done":
            current_text = _rewrite_note_fields(current_text, ocr_redo=False)
            success_keys.append(zotero_key)
        else:
            current_text = _rewrite_note_fields(current_text, ocr_redo=True)
            failed_keys.append(zotero_key)
        note_file.write_text(current_text, encoding="utf-8")
        refresh_index_entry(vault, zotero_key)

    if success_keys:
        print(f"Redo OCR done={len(success_keys)}: {', '.join(success_keys)}", flush=True)
    if failed_keys:
        print(f"Redo OCR pending/failed={len(failed_keys)}: {', '.join(failed_keys)}", flush=True)
    return exit_code


def run_ocr(vault: Path, verbose: bool = False, no_progress: bool = False, selected_keys: set[str] | None = None) -> int:
    from paperforge.pdf_resolver import resolve_pdf_path

    paths = pipeline_paths(vault)
    cleanup_blocked_ocr_dirs(paths)

    # Zombie reset: reset stale processing jobs to pending
    zombie_timeout = int(os.environ.get("PAPERFORGE_ZOMBIE_TIMEOUT_MINUTES", "30"))
    ocr_root = paths.get("ocr")
    if ocr_root and ocr_root.exists():
        for meta_dir in ocr_root.iterdir():
            meta_path = meta_dir / "meta.json"
            if not meta_path.exists():
                continue
            try:
                meta = read_json(meta_path)
            except Exception:
                continue
            zkey = meta.get("zotero_key", meta_dir.name)
            zstatus = str(meta.get("ocr_status", "") or "").strip().lower()
            if zstatus in {"queued", "running"}:
                zstarted = meta.get("ocr_started_at", "")
                if zstarted:
                    try:
                        started_dt = datetime.fromisoformat(zstarted)
                        if (datetime.now(timezone.utc) - started_dt).total_seconds() > zombie_timeout * 60:
                            meta["ocr_status"] = "pending"
                            meta["retry_count"] = int(meta.get("retry_count", 0)) + 1
                            write_json(meta_path, meta)
                            logger.warning(
                                "Zombie reset %s: was %s, started at %s, reset to pending", zkey, zstatus, zstarted
                            )
                    except Exception:
                        pass

    control_actions = load_control_actions(paths)
    target_keys = {key for key, action in control_actions.items() if action.get("do_ocr", False)}
    if selected_keys is not None:
        target_keys &= set(selected_keys)

    target_rows = []
    for export_path in sorted(paths["exports"].glob("*.json")):
        for item in load_export_rows(export_path):
            if item["key"] not in target_keys:
                continue
            pdf_attachments = [a for a in item.get("attachments", []) if a.get("contentType") == "application/pdf"]
            target_rows.append(
                {
                    "zotero_key": item["key"],
                    "has_pdf": bool(pdf_attachments),
                    "pdf_path": pdf_attachments[0]["path"] if pdf_attachments else "",
                }
            )
    for row in target_rows:
        key = row["zotero_key"]
        meta = ensure_ocr_meta(vault, row)
        current = str(meta.get("ocr_status", "") or "").strip().lower()
        if current == "error":
            meta["ocr_status"] = "pending"
            meta["ocr_job_id"] = ""
            meta["ocr_started_at"] = ""
            meta["ocr_finished_at"] = ""
            meta["retry_count"] = 0
            write_json(paths["ocr"] / key / "meta.json", meta)
        elif current == "nopdf":
            meta["ocr_status"] = "pending"
            meta["error"] = ""
            meta["retry_count"] = 0
            write_json(paths["ocr"] / key / "meta.json", meta)
        status, _error = validate_ocr_meta(paths, meta)
        if status == "done_incomplete":
            meta["ocr_status"] = "pending"
            meta["ocr_job_id"] = ""
            meta["ocr_started_at"] = ""
            meta["ocr_finished_at"] = ""
            meta["error"] = _error
            meta["retry_count"] = 0
            write_json(paths["ocr"] / key / "meta.json", meta)
    ocr_queue = sync_ocr_queue(paths, target_rows)
    print(f"DEBUG: target_keys count={len(target_keys)}, target_rows count={len(target_rows)}, queue count={len(ocr_queue)}", flush=True)
    if ocr_queue:
        print(f"DEBUG: first item={ocr_queue[0].get('zotero_key')} status={ocr_queue[0].get('queue_status')}", flush=True)
    max_items_raw = os.environ.get("PADDLEOCR_MAX_ITEMS", "").strip()
    max_items = 3
    if max_items_raw:
        try:
            max_items = max(1, int(max_items_raw))
        except ValueError:
            max_items = 3
    token = os.environ.get("PADDLEOCR_API_TOKEN", "").strip()
    if not token:
        token = os.environ.get("PADDLEOCR_API_TOKEN_USER", "").strip()
    if not token:
        try:
            import winreg

            with winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Environment") as env_key:
                token = str(winreg.QueryValueEx(env_key, "PADDLEOCR_API_TOKEN")[0]).strip()
        except Exception:
            token = ""
    if not token:
        # Fallback: parse vault-root .env
        token = _read_dotenv(vault, "PADDLEOCR_API_TOKEN")
    job_url = os.environ.get("PADDLEOCR_JOB_URL", "https://paddleocr.aistudio-app.com/api/v2/ocr/jobs").strip()
    model = os.environ.get("PADDLEOCR_MODEL", "PaddleOCR-VL-1.6").strip()
    optional_payload = {"useDocOrientationClassify": False, "useDocUnwarping": False, "useChartRecognition": False}
    changed = 0
    active_submitted = 0
    queue_changed = False
    _submitted: set[str] = set()  # keys newly uploaded in this run

    def _do_poll(job_id: str, token_val: str) -> requests.Response:
        resp = requests.get(f"{job_url}/{job_id}", headers={"Authorization": f"bearer {token_val}"}, timeout=60)
        resp.raise_for_status()
        return resp

    for queue_row in progress_bar(ocr_queue, desc="Processing OCR", disable=no_progress):
        key = queue_row["zotero_key"]
        meta = ensure_ocr_meta(vault, queue_row)
        status = str(meta.get("ocr_status", "pending") or "pending").strip().lower()
        queue_row["queue_status"] = status
        if status == "done":
            queue_changed = True
            continue
        if status in {"queued", "running"} and meta.get("ocr_job_id"):
            active_submitted += 1
            if not token:
                continue
            meta_path_poll = paths["ocr"] / key / "meta.json"
            try:
                response = retry_with_meta(_do_poll, meta_path_poll, meta["ocr_job_id"], token)
            except Exception as e:
                meta["ocr_status"] = "pending"
                meta["error"] = str(e)
                meta["last_error"] = str(e)
                meta["retry_count"] = int(meta.get("retry_count", 0)) + 1
                queue_row["queue_status"] = "pending"
                write_json(paths["ocr"] / key / "meta.json", meta)
                changed += 1
                active_submitted = max(0, active_submitted - 1)
                continue
            try:
                payload = response.json()["data"]
                state = payload["state"]
            except (json.JSONDecodeError, KeyError) as e:
                meta["ocr_status"] = "pending"
                meta["error"] = f"API schema mismatch during polling: {e}"
                meta["last_error"] = meta["error"]
                meta["raw_response"] = response.text[:1000]
                meta["retry_count"] = int(meta.get("retry_count", 0)) + 1
                queue_row["queue_status"] = "pending"
                write_json(paths["ocr"] / key / "meta.json", meta)
                changed += 1
                active_submitted = max(0, active_submitted - 1)
                continue
            if state in {"pending", "running"}:
                meta["ocr_status"] = state
                queue_row["queue_status"] = state
                meta["error"] = ""
            elif state == "done":
                try:
                    result_url = payload["resultUrl"]["jsonUrl"]
                    result_response = requests.get(result_url, timeout=120)
                    result_response.raise_for_status()
                    lines = [line.strip() for line in result_response.text.splitlines() if line.strip()]
                    all_results = []
                    for line in lines:
                        page_payload = json.loads(line)["result"]
                        all_results.append(page_payload)
                    page_num, markdown_path, json_path, fulltext_md_path = postprocess_ocr_result(
                        vault, key, all_results
                    )
                except Exception as e:
                    meta["ocr_status"] = "pending"
                    meta["error"] = str(e)
                    meta["retry_count"] = int(meta.get("retry_count", 0)) + 1
                    queue_row["queue_status"] = "pending"
                    write_json(paths["ocr"] / key / "meta.json", meta)
                    changed += 1
                    active_submitted = max(0, active_submitted - 1)
                    continue
                meta["ocr_status"] = "done"
                # Per D-01: auto_analyze_after_ocr opt-in workflow streamlining
                cfg_path = vault / "paperforge.json"
                if cfg_path.exists():
                    try:
                        pf_cfg = read_json(cfg_path)
                        if pf_cfg.get("auto_analyze_after_ocr", False):
                            note_glob = list(paths["literature"].rglob(f"{key}.md"))
                            if not note_glob:
                                note_glob = list(paths["literature"].rglob(f"{key} - *.md"))
                            if note_glob:
                                note_path = max(note_glob, key=lambda p: len(p.parents))
                                text = note_path.read_text(encoding="utf-8")
                                text = re.sub(
                                    r"^analyze:.*$",
                                    "analyze: true",
                                    text,
                                    count=1,
                                    flags=re.MULTILINE,
                                )
                                note_path.write_text(text, encoding="utf-8")
                    except Exception:
                        logger.warning("auto_analyze_after_ocr: failed for %s", key, exc_info=True)
                meta["ocr_finished_at"] = datetime.now(timezone.utc).isoformat()
                meta["page_count"] = page_num
                meta["markdown_path"] = markdown_path
                meta["json_path"] = json_path
                meta["fulltext_md_path"] = fulltext_md_path
                meta["error"] = ""
                queue_row["queue_status"] = "done"
                queue_changed = True
                active_submitted = max(0, active_submitted - 1)
            else:
                meta["ocr_status"] = "error"
                meta["error"] = payload.get("errorMsg", "Unknown OCR failure")
                meta["library_record"] = key
                queue_row["queue_status"] = "error"
                active_submitted = max(0, active_submitted - 1)
            write_json(paths["ocr"] / key / "meta.json", meta)
            changed += 1

    # Upload pending items in batches until none remain (processes all do_ocr items, not just max_items)
    def _do_upload(token_val: str, pdf_path: Path) -> requests.Response:
        with open(pdf_path, "rb") as file_handle:
            resp = requests.post(
                job_url,
                headers={"Authorization": f"bearer {token_val}"},
                data={"model": model, "optionalPayload": json.dumps(optional_payload)},
                files={"file": file_handle},
                timeout=120,
            )
        resp.raise_for_status()
        return resp

    # Combined upload + poll loop: process all items in batches up to max_items concurrency
    import time as _time

    poll_interval = int(os.environ.get("PAPERFORGE_POLL_INTERVAL", "15"))
    max_poll_cycles = int(os.environ.get("PAPERFORGE_POLL_MAX_CYCLES", "60"))
    _completed_count = 0
    _failed_count = 0
    _token_warned = False
    for _cycle in range(max_poll_cycles):
        remaining = [r for r in ocr_queue if r.get("queue_status", "") not in ("done", "nopdf", "blocked")]
        if not remaining:
            break
        available_slots = max(0, max_items - active_submitted)
        if available_slots > 0:
            upload_items = [r for r in remaining if r.get("queue_status", "") not in ("queued", "running")][
                :available_slots
            ]
            for queue_row in upload_items:
                key = queue_row["zotero_key"]
                meta = ensure_ocr_meta(vault, queue_row)
                _sanitized_temp = None
                status = str(meta.get("ocr_status", "pending") or "pending").strip().lower()
                if status in {"done", "queued", "running"}:
                    continue
                resolved_pdf = resolve_pdf_path(
                    queue_row.get("pdf_path", ""),
                    queue_row.get("has_pdf", False),
                    vault,
                    paths.get("zotero_dir") if "zotero_dir" in paths else None,
                )
                if not resolved_pdf:
                    meta["ocr_status"] = "nopdf"
                    queue_row["queue_status"] = "nopdf"
                    write_json(paths["ocr"] / key / "meta.json", meta)
                    changed += 1
                    print(f"OCR: {key} skipped (PDF not found)", flush=True)
                    continue
                if not token:
                    meta["ocr_status"] = "blocked"
                    queue_row["queue_status"] = "blocked"
                    write_json(paths["ocr"] / key / "meta.json", meta)
                    changed += 1
                    if not _token_warned:
                        print("OCR: no API token configured — set PADDLEOCR_API_TOKEN in .env", flush=True)
                        _token_warned = True
                    continue
                upload_pdf = resolved_pdf
                if meta.get("needs_sanitize"):
                    try:
                        import tempfile

                        doc = fitz.open(str(resolved_pdf))
                        _sanitized_temp = Path(tempfile.mktemp(suffix=".pdf"))
                        doc.save(str(_sanitized_temp), garbage=4, deflate=True, clean=True)
                        doc.close()
                        upload_pdf = _sanitized_temp
                        meta["needs_sanitize"] = False
                    except Exception:
                        pass
                if int(meta.get("retry_count", 0)) >= 3:
                    meta["ocr_status"] = "error"
                    meta["error"] = meta.get("error", "") or "Upload failed after 3 retries"
                    queue_row["queue_status"] = "error"
                    write_json(paths["ocr"] / key / "meta.json", meta)
                    changed += 1
                    continue
                print(f"OCR: {key} uploading to PaddleOCR...", flush=True)
                try:
                    response = retry_with_meta(_do_upload, paths["ocr"] / key / "meta.json", token, upload_pdf)
                    meta["ocr_job_id"] = response.json()["data"]["jobId"]
                except Exception as e:
                    if _sanitized_temp is not None and _sanitized_temp.exists():
                        _sanitized_temp.unlink(missing_ok=True)
                    import requests as _requests

                    if isinstance(e, _requests.exceptions.HTTPError):
                        _status = getattr(getattr(e, "response", None), "status_code", 0)
                        if _status == 401:
                            meta["ocr_status"] = "blocked"
                            meta["error"] = "PaddleOCR token invalid"
                            meta["retry_count"] = 3
                            queue_row["queue_status"] = "blocked"
                            write_json(paths["ocr"] / key / "meta.json", meta)
                            changed += 1
                            print(f"OCR: {key} blocked (invalid API token)", flush=True)
                            continue
                    if isinstance(e, FileNotFoundError):
                        meta["ocr_status"] = "nopdf"
                        meta["error"] = "PDF not found"
                        queue_row["queue_status"] = "nopdf"
                        write_json(paths["ocr"] / key / "meta.json", meta)
                        changed += 1
                        continue
                    retry_count = int(meta.get("retry_count", 0)) + 1
                    meta["retry_count"] = retry_count
                    meta["error"] = str(e)
                    meta["last_error"] = str(e)
                    meta["ocr_status"] = "pending"
                    queue_row["queue_status"] = "pending"
                    write_json(paths["ocr"] / key / "meta.json", meta)
                    changed += 1
                    print(f"OCR: {key} upload failed (retry {retry_count}/3): {e}", flush=True)
                    continue
                meta["ocr_status"] = "queued"
                meta["ocr_started_at"] = datetime.now(timezone.utc).isoformat()
                meta["error"] = ""
                queue_row["queue_status"] = "queued"
                write_json(paths["ocr"] / key / "meta.json", meta)
                changed += 1
                active_submitted += 1
                print(f"OCR: {key} queued (job {meta['ocr_job_id']})", flush=True)
            if _sanitized_temp is not None and _sanitized_temp.exists():
                _sanitized_temp.unlink(missing_ok=True)
        poll_items = [r for r in remaining if r.get("queue_status") in ("queued", "running")]
        for queue_row in poll_items:
            key = queue_row["zotero_key"]
            meta = ensure_ocr_meta(vault, queue_row)
            job_id = meta.get("ocr_job_id", "")
            if not job_id:
                continue
            try:
                response = retry_with_meta(_do_poll, paths["ocr"] / key / "meta.json", job_id, token)
                payload = response.json()["data"]
                state = payload["state"]
            except Exception:
                continue
            if state == "done":
                try:
                    result_url = payload["resultUrl"]["jsonUrl"]
                    result_response = requests.get(result_url, timeout=120)
                    if result_response.status_code == 404:
                        meta["ocr_status"] = "pending"
                        meta["ocr_job_id"] = ""
                        meta["needs_sanitize"] = True
                        meta["error"] = "Result object not found on provider (404)"
                        meta["retry_count"] = 0
                        queue_row["queue_status"] = "pending"
                        write_json(paths["ocr"] / key / "meta.json", meta)
                        changed += 1
                        active_submitted = max(0, active_submitted - 1)
                        print(f"OCR: {key} result expired, will retry", flush=True)
                        continue
                    result_response.raise_for_status()
                    lines = [l.strip() for l in result_response.text.splitlines() if l.strip()]
                    results = [json.loads(l)["result"] for l in lines]
                    page_num, md_path, json_path, fulltext_md_path = postprocess_ocr_result(vault, key, results)
                except Exception:
                    continue
                meta["ocr_status"] = "done"
                meta["ocr_finished_at"] = datetime.now(timezone.utc).isoformat()
                meta["page_count"] = page_num
                meta["markdown_path"] = md_path
                meta["json_path"] = json_path
                meta["fulltext_md_path"] = fulltext_md_path
                meta["error"] = ""
                queue_row["queue_status"] = "done"
                queue_changed = True
                active_submitted = max(0, active_submitted - 1)
                _completed_count += 1
                print(f"OCR: {key} completed ({page_num} pages)", flush=True)
            elif state in ("error", "failed"):
                meta["error"] = payload.get("errorMsg", "Unknown OCR failure")
                meta["ocr_status"] = "queued"
                queue_row["queue_status"] = "queued"
                _failed_count += 1
                print(f"OCR: {key} failed: {meta['error']}", flush=True)
            else:
                meta["ocr_status"] = state
                queue_row["queue_status"] = state
            write_json(paths["ocr"] / key / "meta.json", meta)
            changed += 1
        if any(r.get("queue_status") in ("queued", "running") for r in ocr_queue):
            _time.sleep(poll_interval)
    # Collect completed OCR keys for incremental index refresh (before filtering)
    _done_ocr_keys = (
        [r.get("zotero_key", "") for r in ocr_queue if r.get("queue_status") == "done"] if queue_changed else []
    )
    if queue_changed:
        ocr_queue = [row for row in ocr_queue if str(row.get("queue_status", "")).lower() != "done"]
    write_ocr_queue(paths, ocr_queue)
    # Determine exit code: 0 = all settled, 1 = some items still pending
    final_remaining = [r for r in ocr_queue if r.get("queue_status", "") not in ("done", "nopdf", "blocked", "error")]
    pending_keys = [r["zotero_key"] for r in final_remaining]
    final_statuses = {r["zotero_key"]: r.get("queue_status", "?") for r in ocr_queue}
    done_count = sum(1 for s in final_statuses.values() if s == "done")
    failed_count = sum(1 for s in final_statuses.values() if s in ("error", "blocked"))
    pending_count = len(final_remaining)

    summary_parts = []
    if done_count:
        summary_parts.append(f"done={done_count}")
    if failed_count:
        summary_parts.append(f"failed={failed_count}")
    if pending_count:
        summary_parts.append(f"pending={pending_count} ({', '.join(pending_keys)})")
    print(f"OCR: {' '.join(summary_parts) if summary_parts else 'no items processed'}", flush=True)
    if pending_keys:
        print("OCR: re-run to continue polling incomplete items", flush=True)

    try:
        _sync.run_selection_sync(vault)
        if _done_ocr_keys:
            done_keys = [k for k in _done_ocr_keys if k]
            for ocr_key in done_keys:
                refresh_index_entry(vault, ocr_key)
            if verbose:
                print(f"ocr: refreshed {len(done_keys)} index entries incrementally")
        else:
            _sync.run_index_refresh(vault)
    except ImportError:
        _sync.run_index_refresh(vault)
    except Exception as e:
        logger.error("Post-OCR index refresh failed: %s", e)
    print(f"ocr: updated {changed} records")
    return 1 if pending_keys else 0