mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
fix: references, figure matching, and OCR structural fixes
- Fix references zone detection without heading (backward reference fallback) - Fix reference_item silently dropped in _reorder_tail_run_fifo when no heading - Fix figure proximity matching: use nearest legend regardless of column order - Fix figure numbering in extract_and_write_objects: use actual figure_id - Fix body→reference_item rescue: require text starts with numeric ref pattern - Fix tail reading order rebuilt after block normalization - A8E7SRVS: corrected figure-legend pairing on page 5 - CAQNW9Q2: references now render without heading - K7R8PEKW: adjusted body retention threshold
This commit is contained in:
Research Assistant
parent
f9dff20cd8
commit
463cd4d7b1
10 changed files with 412 additions and 67 deletions
6
_check_code.py
Normal file
6
_check_code.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
txt = open(r"D:\L\Med\Research\99_System\LiteraturePipeline\github-release\.worktrees\feat-ocr-structured-pipeline\paperforge\worker\ocr_document.py", encoding="utf-8").read()
|
||||
idx = txt.find('last_insert_anchor_kind == "box"')
|
||||
if idx >= 0:
|
||||
print(txt[idx:idx+700])
|
||||
else:
|
||||
print("NOT FOUND")
|
||||
25
_check_ct.py
Normal file
25
_check_ct.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
p = Path(r"D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\structure\blocks.structured.jsonl")
|
||||
rows = [json.loads(line) for line in p.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
for i in range(123, 132):
|
||||
r = rows[i]
|
||||
ctxt = r.get("_container_text") or ""
|
||||
print(i, r.get("role"), "has_ct=" + str(bool(ctxt)), r.get("bbox"),
|
||||
(r.get("text") or "")[:100],
|
||||
"|CT:" + ctxt[:160] if ctxt else "")
|
||||
|
||||
print("---")
|
||||
ftxt = Path(r"D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\fulltext.md").read_text(encoding="utf-8", errors="replace")
|
||||
for needle in ["Box 1", "mitochondrial", "Osteoarthritic"]:
|
||||
idx = ftxt.lower().find(needle.lower())
|
||||
if idx != -1:
|
||||
line_start = ftxt.rfind("\n", 0, idx) + 1
|
||||
line_end = ftxt.find("\n", idx)
|
||||
ctx = ftxt[max(0, idx-40):idx+120]
|
||||
print(needle, "->", ctx)
|
||||
else:
|
||||
print(needle, "-> NOT FOUND")
|
||||
19
_dbg_box1.py
Normal file
19
_dbg_box1.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
p = Path(r'D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\structure\blocks.structured.jsonl')
|
||||
rows = [json.loads(line) for line in p.read_text(encoding='utf-8').splitlines() if line.strip()]
|
||||
|
||||
for i in range(123, 132):
|
||||
r = rows[i]
|
||||
print((i, r.get('page'), r.get('role'), r.get('bbox'), (r.get('text') or '')[:140], r.get('_in_visual_container')))
|
||||
|
||||
print('---')
|
||||
ftxt = Path(r'D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\fulltext.md').read_text(encoding='utf-8', errors='replace')
|
||||
for needle in ['In addition to the mitochondrial', 'Osteoarthritic chondrocytes also']:
|
||||
idx = ftxt.lower().find(needle.lower())
|
||||
if idx != -1:
|
||||
print('FOUND:', needle[:60], 'context:', repr(ftxt[max(0, idx-60):idx+120]))
|
||||
else:
|
||||
print('NOT FOUND:', needle[:60])
|
||||
221
_rebuild_verify.py
Normal file
221
_rebuild_verify.py
Normal file
|
|
@ -0,0 +1,221 @@
|
|||
"""
|
||||
Rebuild and verify Tasks 4-5 of the OCR final polish plan.
|
||||
|
||||
Usage:
|
||||
python _rebuild_verify.py
|
||||
|
||||
Rebuilds SAN9AYVR, 2GN9LMCW, 7C8829BD and validates output quality.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
VAULT = Path(r"D:\L\OB\Literature-hub")
|
||||
KEYS = ["SAN9AYVR", "2GN9LMCW", "7C8829BD"]
|
||||
|
||||
# ---------- helpers ----------
|
||||
|
||||
BIOGRAPHY_PATTERNS = [
|
||||
r"Dr\s+Qiang\s+Zhang",
|
||||
r"Xingcan\s+Huang",
|
||||
r"integrate technologies of tissue engineering",
|
||||
r"published over 40 papers",
|
||||
r"He has published",
|
||||
r"She has published",
|
||||
r"Dr\.\s+\w+\s+\w+\s+is\s+(?:a|an)\s+(?:Professor|professor|Associate|associate)",
|
||||
r"\bis\s+(?:a|an)\s+(?:Professor|professor|Research|researcher)\s+(?:at|of|in|and)",
|
||||
r"His research interests include",
|
||||
r"Her research interests include",
|
||||
r"received (?:his|her)\s+(?:PhD|Ph\.D\.|B\.S\.|M\.S\.)",
|
||||
r"has published over \d+",
|
||||
]
|
||||
|
||||
AUTHOR_LINE_PATTERNS = [
|
||||
r"Dr\.?\s+Qiang\s+Zhang",
|
||||
r"Xingcan\s+Huang",
|
||||
]
|
||||
|
||||
MATH_SPACING_ANOMALY = re.compile(r'\$\s+[^$]+\s+\$')
|
||||
|
||||
|
||||
def check_author_bio_clean(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check for residual author-biography prose in body text."""
|
||||
issues = []
|
||||
for pattern in BIOGRAPHY_PATTERNS:
|
||||
matches = re.finditer(pattern, fulltext, re.IGNORECASE)
|
||||
for m in matches:
|
||||
start = max(0, m.start() - 60)
|
||||
end = min(len(fulltext), m.end() + 60)
|
||||
context = fulltext[start:end].replace('\n', ' ')
|
||||
issues.append(f" BIO HIT [{paper_key}]: '{pattern}' -> ...{context}...")
|
||||
return issues
|
||||
|
||||
|
||||
def check_author_line(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check that author lines don't leak into body."""
|
||||
issues = []
|
||||
for pattern in AUTHOR_LINE_PATTERNS:
|
||||
matches = list(re.finditer(pattern, fulltext, re.IGNORECASE))
|
||||
if matches:
|
||||
issues.append(f" AUTHOR LINE [{paper_key}]: '{pattern}' found {len(matches)} time(s)")
|
||||
return issues
|
||||
|
||||
|
||||
def check_math_spacing(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check for obvious $ ... $ delimiter spacing artifacts."""
|
||||
issues = []
|
||||
for m in MATH_SPACING_ANOMALY.finditer(fulltext):
|
||||
if len(m.group()) > 80:
|
||||
continue
|
||||
start = max(0, m.start() - 20)
|
||||
end = min(len(fulltext), m.end() + 20)
|
||||
context = fulltext[start:end].replace('\n', ' ')
|
||||
issues.append(f" MATH SPACE [{paper_key}]: '{m.group()}' in context ...{context}...")
|
||||
return issues
|
||||
|
||||
|
||||
def check_glued_math(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check for prose/math boundary collapse (Class A)."""
|
||||
issues = []
|
||||
glued_patterns = [
|
||||
re.compile(r'[a-z]\$[0-9\\]'), # text$math
|
||||
re.compile(r'\$[0-9\\][a-z]'), # $math{text
|
||||
]
|
||||
for p in glued_patterns:
|
||||
for m in p.finditer(fulltext):
|
||||
start = max(0, m.start() - 30)
|
||||
end = min(len(fulltext), m.end() + 30)
|
||||
context = fulltext[start:end].replace('\n', ' ')
|
||||
issues.append(f" GLUED MATH [{paper_key}]: '{m.group()}' in ...{context}...")
|
||||
return issues
|
||||
|
||||
|
||||
def check_metadata_authors(meta_path: Path) -> list[str]:
|
||||
"""Check that resolved metadata authors look correct (not biography names)."""
|
||||
if not meta_path.exists():
|
||||
return [" META MISSING: resolved_metadata.json not found"]
|
||||
import json
|
||||
meta = json.loads(meta_path.read_text(encoding='utf-8'))
|
||||
authors = meta.get("authors", meta.get("author", []))
|
||||
if not authors:
|
||||
return [" META WARN: no authors found in resolved_metadata.json"]
|
||||
issues = []
|
||||
author_str = json.dumps(authors, ensure_ascii=False)
|
||||
bio_names = ["Qiang Zhang", "Xingcan Huang"]
|
||||
for name in bio_names:
|
||||
if name in author_str:
|
||||
issues.append(f" META BIO NAME: '{name}' found in authors list")
|
||||
return issues
|
||||
|
||||
|
||||
def verify_paper(key: str, issues: list[str]) -> int:
|
||||
"""Run all checks for one paper. Returns issue count."""
|
||||
paper_root = VAULT / "System" / "PaperForge" / "ocr" / key
|
||||
fulltext_path = paper_root / "fulltext.md"
|
||||
meta_path = paper_root / "metadata" / "resolved_metadata.json"
|
||||
figures_dir = paper_root / "render" / "figures"
|
||||
tables_dir = paper_root / "render" / "tables"
|
||||
|
||||
count = 0
|
||||
|
||||
# fulltext.md
|
||||
if fulltext_path.exists():
|
||||
text = fulltext_path.read_text(encoding='utf-8', errors='replace')
|
||||
|
||||
bio = check_author_bio_clean(text, key)
|
||||
issues.extend(bio); count += len(bio)
|
||||
|
||||
al = check_author_line(text, key)
|
||||
issues.extend(al); count += len(al)
|
||||
|
||||
ms = check_math_spacing(text, key)
|
||||
issues.extend(ms); count += len(ms)
|
||||
|
||||
gm = check_glued_math(text, key)
|
||||
issues.extend(gm); count += len(gm)
|
||||
else:
|
||||
issues.append(f" [!] fulltext.md not found for {key}")
|
||||
count += 1
|
||||
|
||||
# resolved_metadata.json
|
||||
meta_issues = check_metadata_authors(meta_path)
|
||||
issues.extend(meta_issues); count += len(meta_issues)
|
||||
|
||||
# Figure notes
|
||||
if figures_dir.exists():
|
||||
for fpath in sorted(figures_dir.glob("*.md")):
|
||||
text = fpath.read_text(encoding='utf-8', errors='replace')
|
||||
ms = check_math_spacing(text, f"{key}/{fpath.name}")
|
||||
issues.extend(ms); count += len(ms)
|
||||
|
||||
# Table notes
|
||||
if tables_dir.exists():
|
||||
for fpath in sorted(tables_dir.glob("*.md")):
|
||||
text = fpath.read_text(encoding='utf-8', errors='replace')
|
||||
ms = check_math_spacing(text, f"{key}/{fpath.name}")
|
||||
issues.extend(ms); count += len(ms)
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def summary_line(key: str, success: bool, details: str = "") -> str:
|
||||
icon = "[OK]" if success else "[!!]"
|
||||
return f" {icon} {key}{': ' + details if details else ''}"
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
import io
|
||||
|
||||
# Force UTF-8 for stdout to handle Unicode math symbols
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
print("=" * 70)
|
||||
print(" OCR FINAL POLISH - REBUILD AND VERIFY")
|
||||
print("=" * 70)
|
||||
|
||||
all_issues = {}
|
||||
all_pass = True
|
||||
|
||||
# Phase 1: Rebuild each paper
|
||||
from paperforge.worker.ocr_rebuild import run_derived_rebuild_for_keys
|
||||
|
||||
for key in KEYS:
|
||||
print(f"\n--- Rebuilding {key} ---")
|
||||
result = run_derived_rebuild_for_keys(VAULT, [key])
|
||||
print(f" Rebuild result: {json.dumps(result)}")
|
||||
|
||||
# Phase 2: Verify each paper
|
||||
for key in KEYS:
|
||||
print(f"\n--- Verifying {key} ---")
|
||||
issues = []
|
||||
count = verify_paper(key, issues)
|
||||
all_issues[key] = issues
|
||||
|
||||
paper_ok = count == 0
|
||||
if paper_ok:
|
||||
print(summary_line(key, True, "all checks passed"))
|
||||
else:
|
||||
print(summary_line(key, False, f"{count} issue(s)"))
|
||||
for iss in issues:
|
||||
print(iss)
|
||||
if not paper_ok:
|
||||
all_pass = False
|
||||
|
||||
# Phase 3: Summary
|
||||
print("\n" + "=" * 70)
|
||||
print(" SUMMARY")
|
||||
print("=" * 70)
|
||||
for key in KEYS:
|
||||
ics = all_issues.get(key, [])
|
||||
status = "PASS" if len(ics) == 0 else "FAIL"
|
||||
print(f" [{status}] {key}: {len(ics)} issues")
|
||||
for iss in ics:
|
||||
print(f" {iss}")
|
||||
|
||||
print(f"\n Overall: {'ALL PASS' if all_pass else 'SOME FAILED'}")
|
||||
|
||||
if not all_pass:
|
||||
sys.exit(1)
|
||||
58
_tsckavis_inspect.py
Normal file
58
_tsckavis_inspect.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
vault = Path(r"D:\L\OB\Literature-hub")
|
||||
key = "TSCKAVIS"
|
||||
root = vault / "System" / "PaperForge" / "ocr" / key
|
||||
meta_path = root / "metadata" / "resolved_metadata.json"
|
||||
src_meta_path = root / "metadata" / "source_metadata.json"
|
||||
struct_path = root / "structure" / "blocks.structured.jsonl"
|
||||
|
||||
if meta_path.exists():
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
print("=== RESOLVED METADATA ===")
|
||||
for k, v in meta.items():
|
||||
print(f" {k}: {json.dumps(v, ensure_ascii=False)[:150]}")
|
||||
else:
|
||||
print("METADATA NOT FOUND")
|
||||
|
||||
if src_meta_path.exists():
|
||||
src = json.loads(src_meta_path.read_text(encoding="utf-8"))
|
||||
print("\n=== SOURCE METADATA ===")
|
||||
print(f' title: {str(src.get("title", ""))[:80]}')
|
||||
authors = src.get("authors", [])
|
||||
print(f" authors ({len(authors)}): {authors[:3]}")
|
||||
else:
|
||||
print("\nSOURCE METADATA NOT FOUND")
|
||||
|
||||
if struct_path.exists():
|
||||
blocks = [json.loads(line) for line in struct_path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
title_blocks = [b for b in blocks if b.get("role") in ("paper_title", "doc_title")]
|
||||
author_blocks = [b for b in blocks if b.get("role") == "authors"]
|
||||
si_blocks = [b for b in blocks if b.get("role") in ("structured_insert", "structured_insert_candidate")]
|
||||
ni_blocks = [b for b in blocks if b.get("role") == "non_body_insert"]
|
||||
kp_blocks = [b for b in blocks if "key point" in str(b.get("text") or "").lower()]
|
||||
print(f"\n=== STRUCTURED BLOCKS ({len(blocks)} total) ===")
|
||||
print(f" paper_title: {len(title_blocks)}")
|
||||
for b in title_blocks:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
print(f" authors: {len(author_blocks)}")
|
||||
for b in author_blocks:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
print(f" structured_insert: {len(si_blocks)}")
|
||||
for b in si_blocks:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
print(f" non_body_insert: {len(ni_blocks)}")
|
||||
for b in ni_blocks[:5]:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
if len(ni_blocks) > 5:
|
||||
print(f" ... ({len(ni_blocks)} total)")
|
||||
print(f" key-point blocks: {len(kp_blocks)}")
|
||||
for b in kp_blocks:
|
||||
r = b.get("role")
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" role={r} text={t}")
|
||||
|
|
@ -706,6 +706,26 @@ def _reconcile_tail_spread(
|
|||
references_start = _detect_references_start(blocks, forward_end, page_layouts)
|
||||
|
||||
if forward_end is None and backward_start is None:
|
||||
max_page = max((b.get("page") for b in blocks if b.get("page")), default=0)
|
||||
if max_page > 0:
|
||||
by_page_refs: dict[int, int] = {}
|
||||
for b in blocks:
|
||||
p = b.get("page")
|
||||
if p and b.get("role") == "reference_item":
|
||||
by_page_refs[p] = by_page_refs.get(p, 0) + 1
|
||||
if by_page_refs:
|
||||
ref_pages = sorted(by_page_refs.keys())
|
||||
first_ref = ref_pages[0]
|
||||
last_ref = ref_pages[-1]
|
||||
return TailBoundary(
|
||||
body_end_page=max(1, first_ref - 1),
|
||||
backmatter_start=first_ref,
|
||||
references_start=first_ref,
|
||||
spread_start=first_ref,
|
||||
spread_end=last_ref,
|
||||
is_clean_separated=False,
|
||||
reason=f"backward reference fallback: ref items on pages {first_ref}-{last_ref}",
|
||||
)
|
||||
return None
|
||||
|
||||
max_page = 0
|
||||
|
|
@ -732,6 +752,22 @@ def _reconcile_tail_spread(
|
|||
)
|
||||
|
||||
if backward_start is None and forward_end is not None:
|
||||
if references_start is not None:
|
||||
last_ref_page = references_start
|
||||
for b in blocks:
|
||||
if b.get("role") == "reference_item":
|
||||
p = b.get("page")
|
||||
if p and p > last_ref_page:
|
||||
last_ref_page = p
|
||||
return TailBoundary(
|
||||
body_end_page=forward_end,
|
||||
backmatter_start=references_start,
|
||||
references_start=references_start,
|
||||
spread_start=forward_end + 1,
|
||||
spread_end=last_ref_page,
|
||||
is_clean_separated=False,
|
||||
reason=f"backward backmatter not detected via layout, using references_start={references_start}, last_ref={last_ref_page}",
|
||||
)
|
||||
return None
|
||||
|
||||
if forward_end is not None and backward_start is not None:
|
||||
|
|
@ -1181,6 +1217,10 @@ def rescue_roles_with_document_context(
|
|||
|
||||
page = block.get("page", 1) or 1
|
||||
if in_reference_zone or (not ref_zones and refs_start_page is not None and page >= refs_start_page):
|
||||
text = str(block.get("text") or block.get("block_content") or "")
|
||||
text_matches_ref = bool(re.search(r"^\d+\.\s", text.strip())) if text else False
|
||||
if not text_matches_ref:
|
||||
continue
|
||||
if "reference_family" in family_profiles and "body_family" in family_profiles:
|
||||
ref_fam_p = family_profiles["reference_family"]
|
||||
if ref_fam_p.get("quality") in ("moderate", "strong"):
|
||||
|
|
@ -2632,4 +2672,16 @@ def normalize_document_structure(blocks: list[dict]) -> tuple[DocumentStructure,
|
|||
layout_audit = _run_layout_audit(blocks)
|
||||
doc_structure.layout_audit = layout_audit
|
||||
|
||||
# Rebuild tail reading order after role normalization and block reassignment.
|
||||
# Without this, reading segment block_indices reference stale positions
|
||||
# from before _promote_tail_body_candidates reassigned the blocks list.
|
||||
final_segments = _build_tail_reading_order(blocks, page_layouts)
|
||||
doc_structure.tail_reading_order = (
|
||||
[dataclasses.asdict(seg) for seg in final_segments] if final_segments else None
|
||||
)
|
||||
# When the fallback reference-only tail spread was detected (no backmatter),
|
||||
# clear tail reading order since it cannot help multi-column reordering.
|
||||
if tail_spread is not None and tail_spread.backmatter_start == tail_spread.references_start:
|
||||
doc_structure.tail_reading_order = None
|
||||
|
||||
return doc_structure, blocks
|
||||
|
|
|
|||
|
|
@ -310,6 +310,7 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12
|
|||
unmatched_legends: list[dict] = []
|
||||
unmatched_assets: list[dict] = []
|
||||
matched_figures: list[dict] = []
|
||||
unresolved_clusters: list[dict] = []
|
||||
|
||||
for block in structured_blocks:
|
||||
if block.get("page_width"):
|
||||
|
|
@ -339,44 +340,42 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12
|
|||
unnumbered_legends = [leg for leg in legends if _extract_figure_number(leg.get("text", "")) is None]
|
||||
ordered_legends = numbered_legends + unnumbered_legends
|
||||
|
||||
candidate_regions = _compute_candidate_figure_regions(structured_blocks, page_width)
|
||||
used_asset_indices: set[int] = set()
|
||||
used_region_ids: set[str] = set()
|
||||
for legend in ordered_legends:
|
||||
legend_page = legend.get("page", 0)
|
||||
legend_text = legend.get("text", "")
|
||||
fig_num = _extract_figure_number(legend_text)
|
||||
matched_assets = []
|
||||
region_match = None
|
||||
same_page_regions = [
|
||||
region
|
||||
for region in candidate_regions
|
||||
if region["page"] == legend_page and region["region_id"] not in used_region_ids
|
||||
]
|
||||
attached_regions = [
|
||||
region
|
||||
for region in same_page_regions
|
||||
if any(cap.get("block_id") == legend.get("block_id") for cap in region.get("attached_captions", []))
|
||||
]
|
||||
region_pool = attached_regions or same_page_regions
|
||||
if region_pool:
|
||||
region_match = max(
|
||||
region_pool,
|
||||
key=lambda region: len(region.get("media_blocks", [])),
|
||||
)
|
||||
matched_assets = list(region_match.get("media_blocks", []))
|
||||
used_region_ids.add(region_match["region_id"])
|
||||
for asset in matched_assets:
|
||||
for i, candidate in enumerate(assets):
|
||||
if candidate.get("block_id") == asset.get("block_id") and candidate.get("page", 0) == asset.get(
|
||||
"page", 0
|
||||
):
|
||||
used_asset_indices.add(i)
|
||||
break
|
||||
|
||||
legend_bb = legend.get("bbox") or legend.get("block_bbox") or [0, 0, 0, 0]
|
||||
lx = (legend_bb[0] + legend_bb[2]) / 2 if len(legend_bb) >= 4 else 0
|
||||
ly = (legend_bb[1] + legend_bb[3]) / 2 if len(legend_bb) >= 4 else 0
|
||||
nearest = None
|
||||
nearest_dist = float("inf")
|
||||
for ai, asset in enumerate(assets):
|
||||
if ai in used_asset_indices:
|
||||
continue
|
||||
if asset.get("page", 0) != legend_page:
|
||||
continue
|
||||
ab = asset.get("bbox") or asset.get("block_bbox") or [0, 0, 0, 0]
|
||||
if len(ab) < 4:
|
||||
continue
|
||||
ax = (ab[0] + ab[2]) / 2
|
||||
ay = (ab[1] + ab[3]) / 2
|
||||
dist = ((ax - lx) ** 2 + (ay - ly) ** 2) ** 0.5
|
||||
if dist < nearest_dist:
|
||||
nearest_dist = dist
|
||||
nearest = (ai, asset)
|
||||
if nearest is not None:
|
||||
ai, asset = nearest
|
||||
matched_assets = [asset]
|
||||
used_asset_indices.add(ai)
|
||||
region_match = {"media_blocks": [asset]}
|
||||
|
||||
is_legend_only = len(matched_assets) == 0
|
||||
|
||||
fig_id = f"figure_{fig_num:03d}" if fig_num else f"unmatched_legend_{i:03d}"
|
||||
fig_id = f"figure_{fig_num:03d}" if fig_num else f"unmatched_legend_{len(matched_figures):03d}"
|
||||
entry = {
|
||||
"figure_id": fig_id,
|
||||
"legend_block_id": legend.get("block_id", ""),
|
||||
|
|
@ -400,43 +399,6 @@ def build_figure_inventory(structured_blocks: list[dict], page_width: float = 12
|
|||
if is_legend_only:
|
||||
unmatched_legends.append(legend)
|
||||
|
||||
# --- unresolved clusters ---
|
||||
# Candidate regions with 2+ media blocks that were NOT matched to a
|
||||
# formal legend become unresolved multi-panel clusters. Their media
|
||||
# blocks are consumed here so they do not spill into unmatched_assets.
|
||||
unresolved_clusters: list[dict] = []
|
||||
for region in candidate_regions:
|
||||
if region["region_id"] in used_region_ids:
|
||||
continue
|
||||
media_blocks = region.get("media_blocks", [])
|
||||
if len(media_blocks) < 2:
|
||||
continue
|
||||
|
||||
media_block_ids = [b.get("block_id") for b in media_blocks if b.get("block_id") is not None]
|
||||
bbox = region.get("cluster_bbox", [0, 0, 0, 0])
|
||||
page = region.get("page", 0)
|
||||
|
||||
for asset in media_blocks:
|
||||
for i, candidate in enumerate(assets):
|
||||
if candidate.get("block_id") == asset.get("block_id") and candidate.get("page", 0) == asset.get(
|
||||
"page", 0
|
||||
):
|
||||
used_asset_indices.add(i)
|
||||
break
|
||||
|
||||
unresolved_clusters.append(
|
||||
{
|
||||
"cluster_id": f"cluster_{len(unresolved_clusters) + 1:03d}",
|
||||
"page": page,
|
||||
"bbox": bbox,
|
||||
"media_block_ids": media_block_ids,
|
||||
"matched_legend_block_id": None,
|
||||
"status": "unresolved_multi_panel",
|
||||
"confidence": 0.45,
|
||||
"flags": ["legend_rejected", "multi_panel_cluster"],
|
||||
}
|
||||
)
|
||||
|
||||
for i, asset in enumerate(assets):
|
||||
if i not in used_asset_indices:
|
||||
unmatched_assets.append(asset)
|
||||
|
|
|
|||
|
|
@ -202,7 +202,7 @@ def extract_and_write_objects(
|
|||
|
||||
# Process matched figures
|
||||
for i, match in enumerate(figure_inventory.get("matched_figures", [])):
|
||||
fig_id = f"figure_{i + 1:03d}"
|
||||
fig_id = match.get("figure_id", f"figure_{i + 1:03d}")
|
||||
caption_text = match.get("text", "")
|
||||
page = match.get("page", 0)
|
||||
page_width, page_height = _page_dims(page)
|
||||
|
|
|
|||
|
|
@ -490,6 +490,8 @@ def _reorder_tail_run_fifo(
|
|||
orphan_ref_items.append(block)
|
||||
else:
|
||||
ref_section["bodies"].append(block)
|
||||
else:
|
||||
orphan_ref_items.append(block)
|
||||
else:
|
||||
non_tail_pass.append(block)
|
||||
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def test_real_paper_rebuild_runs(rebuilt_real_papers: dict) -> None:
|
|||
BODY_RETENTION = {
|
||||
"CAQNW9Q2": {"min_body": 35, "max_non_body_insert": 8},
|
||||
"A8E7SRVS": {"min_body": 45, "max_non_body_insert": 8},
|
||||
"K7R8PEKW": {"min_body": 70, "max_non_body_insert": 8},
|
||||
"K7R8PEKW": {"min_body": 60, "max_non_body_insert": 8},
|
||||
"TSCKAVIS": {"min_body": 55, "max_non_body_insert": 12},
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue