mirror of
https://github.com/lllin000/PaperForge.git
synced 2026-07-22 06:50:53 +00:00
chore: remove root debug scripts, relocate rebuild_verify to scripts/dev
This commit is contained in:
Research Assistant
parent
c036ef7670
commit
04c942c8db
5 changed files with 34 additions and 137 deletions
|
|
@ -1,6 +0,0 @@
|
|||
txt = open(r"D:\L\Med\Research\99_System\LiteraturePipeline\github-release\.worktrees\feat-ocr-structured-pipeline\paperforge\worker\ocr_document.py", encoding="utf-8").read()
|
||||
idx = txt.find('last_insert_anchor_kind == "box"')
|
||||
if idx >= 0:
|
||||
print(txt[idx:idx+700])
|
||||
else:
|
||||
print("NOT FOUND")
|
||||
25
_check_ct.py
25
_check_ct.py
|
|
@ -1,25 +0,0 @@
|
|||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
|
||||
p = Path(r"D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\structure\blocks.structured.jsonl")
|
||||
rows = [json.loads(line) for line in p.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
|
||||
for i in range(123, 132):
|
||||
r = rows[i]
|
||||
ctxt = r.get("_container_text") or ""
|
||||
print(i, r.get("role"), "has_ct=" + str(bool(ctxt)), r.get("bbox"),
|
||||
(r.get("text") or "")[:100],
|
||||
"|CT:" + ctxt[:160] if ctxt else "")
|
||||
|
||||
print("---")
|
||||
ftxt = Path(r"D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\fulltext.md").read_text(encoding="utf-8", errors="replace")
|
||||
for needle in ["Box 1", "mitochondrial", "Osteoarthritic"]:
|
||||
idx = ftxt.lower().find(needle.lower())
|
||||
if idx != -1:
|
||||
line_start = ftxt.rfind("\n", 0, idx) + 1
|
||||
line_end = ftxt.find("\n", idx)
|
||||
ctx = ftxt[max(0, idx-40):idx+120]
|
||||
print(needle, "->", ctx)
|
||||
else:
|
||||
print(needle, "-> NOT FOUND")
|
||||
19
_dbg_box1.py
19
_dbg_box1.py
|
|
@ -1,19 +0,0 @@
|
|||
import json, sys
|
||||
from pathlib import Path
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
p = Path(r'D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\structure\blocks.structured.jsonl')
|
||||
rows = [json.loads(line) for line in p.read_text(encoding='utf-8').splitlines() if line.strip()]
|
||||
|
||||
for i in range(123, 132):
|
||||
r = rows[i]
|
||||
print((i, r.get('page'), r.get('role'), r.get('bbox'), (r.get('text') or '')[:140], r.get('_in_visual_container')))
|
||||
|
||||
print('---')
|
||||
ftxt = Path(r'D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\fulltext.md').read_text(encoding='utf-8', errors='replace')
|
||||
for needle in ['In addition to the mitochondrial', 'Osteoarthritic chondrocytes also']:
|
||||
idx = ftxt.lower().find(needle.lower())
|
||||
if idx != -1:
|
||||
print('FOUND:', needle[:60], 'context:', repr(ftxt[max(0, idx-60):idx+120]))
|
||||
else:
|
||||
print('NOT FOUND:', needle[:60])
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
vault = Path(r"D:\L\OB\Literature-hub")
|
||||
key = "TSCKAVIS"
|
||||
root = vault / "System" / "PaperForge" / "ocr" / key
|
||||
meta_path = root / "metadata" / "resolved_metadata.json"
|
||||
src_meta_path = root / "metadata" / "source_metadata.json"
|
||||
struct_path = root / "structure" / "blocks.structured.jsonl"
|
||||
|
||||
if meta_path.exists():
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
print("=== RESOLVED METADATA ===")
|
||||
for k, v in meta.items():
|
||||
print(f" {k}: {json.dumps(v, ensure_ascii=False)[:150]}")
|
||||
else:
|
||||
print("METADATA NOT FOUND")
|
||||
|
||||
if src_meta_path.exists():
|
||||
src = json.loads(src_meta_path.read_text(encoding="utf-8"))
|
||||
print("\n=== SOURCE METADATA ===")
|
||||
print(f' title: {str(src.get("title", ""))[:80]}')
|
||||
authors = src.get("authors", [])
|
||||
print(f" authors ({len(authors)}): {authors[:3]}")
|
||||
else:
|
||||
print("\nSOURCE METADATA NOT FOUND")
|
||||
|
||||
if struct_path.exists():
|
||||
blocks = [json.loads(line) for line in struct_path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
||||
title_blocks = [b for b in blocks if b.get("role") in ("paper_title", "doc_title")]
|
||||
author_blocks = [b for b in blocks if b.get("role") == "authors"]
|
||||
si_blocks = [b for b in blocks if b.get("role") in ("structured_insert", "structured_insert_candidate")]
|
||||
ni_blocks = [b for b in blocks if b.get("role") == "non_body_insert"]
|
||||
kp_blocks = [b for b in blocks if "key point" in str(b.get("text") or "").lower()]
|
||||
print(f"\n=== STRUCTURED BLOCKS ({len(blocks)} total) ===")
|
||||
print(f" paper_title: {len(title_blocks)}")
|
||||
for b in title_blocks:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
print(f" authors: {len(author_blocks)}")
|
||||
for b in author_blocks:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
print(f" structured_insert: {len(si_blocks)}")
|
||||
for b in si_blocks:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
print(f" non_body_insert: {len(ni_blocks)}")
|
||||
for b in ni_blocks[:5]:
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" -> {t}")
|
||||
if len(ni_blocks) > 5:
|
||||
print(f" ... ({len(ni_blocks)} total)")
|
||||
print(f" key-point blocks: {len(kp_blocks)}")
|
||||
for b in kp_blocks:
|
||||
r = b.get("role")
|
||||
t = (b.get("text") or "")[:80]
|
||||
print(f" role={r} text={t}")
|
||||
|
|
@ -1,17 +1,40 @@
|
|||
"""
|
||||
Rebuild and verify Tasks 4-5 of the OCR final polish plan.
|
||||
Rebuild and verify OCR output quality for specified papers.
|
||||
|
||||
Usage:
|
||||
python _rebuild_verify.py
|
||||
set PAPERFORGE_REAL_OCR_VAULT=D:\L\OB\Literature-hub
|
||||
set PAPERFORGE_REAL_OCR_KEYS=SAN9AYVR,2GN9LMCW,7C8829BD
|
||||
python scripts/dev/ocr_rebuild_verify.py
|
||||
|
||||
Rebuilds SAN9AYVR, 2GN9LMCW, 7C8829BD and validates output quality.
|
||||
Environment Variables:
|
||||
PAPERFORGE_REAL_OCR_VAULT - Path to the Obsidian vault (required)
|
||||
PAPERFORGE_REAL_OCR_KEYS - Comma-separated paper keys to verify (required)
|
||||
"""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
VAULT = Path(r"D:\L\OB\Literature-hub")
|
||||
KEYS = ["SAN9AYVR", "2GN9LMCW", "7C8829BD"]
|
||||
vault = Path(os.environ.get("PAPERFORGE_REAL_OCR_VAULT", ""))
|
||||
keys = os.environ.get("PAPERFORGE_REAL_OCR_KEYS", "").split(",")
|
||||
keys = [k.strip() for k in keys if k.strip()]
|
||||
|
||||
if not vault or not vault.exists():
|
||||
print("[!!] PAPERFORGE_REAL_OCR_VAULT is not set or does not exist.")
|
||||
print(" Set it to a valid Obsidian vault root, e.g.:")
|
||||
print(' set PAPERFORGE_REAL_OCR_VAULT=D:\\L\\OB\\Literature-hub')
|
||||
sys.exit(1)
|
||||
|
||||
if not keys:
|
||||
print("[!!] PAPERFORGE_REAL_OCR_KEYS is not set or empty.")
|
||||
print(" Set it to comma-separated paper keys, e.g.:")
|
||||
print(" set PAPERFORGE_REAL_OCR_KEYS=SAN9AYVR,2GN9LMCW,7C8829BD")
|
||||
sys.exit(1)
|
||||
|
||||
VAULT = vault
|
||||
KEYS = keys
|
||||
|
||||
# ---------- helpers ----------
|
||||
|
||||
|
|
@ -39,7 +62,6 @@ MATH_SPACING_ANOMALY = re.compile(r'\$\s+[^$]+\s+\$')
|
|||
|
||||
|
||||
def check_author_bio_clean(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check for residual author-biography prose in body text."""
|
||||
issues = []
|
||||
for pattern in BIOGRAPHY_PATTERNS:
|
||||
matches = re.finditer(pattern, fulltext, re.IGNORECASE)
|
||||
|
|
@ -52,7 +74,6 @@ def check_author_bio_clean(fulltext: str, paper_key: str) -> list[str]:
|
|||
|
||||
|
||||
def check_author_line(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check that author lines don't leak into body."""
|
||||
issues = []
|
||||
for pattern in AUTHOR_LINE_PATTERNS:
|
||||
matches = list(re.finditer(pattern, fulltext, re.IGNORECASE))
|
||||
|
|
@ -62,7 +83,6 @@ def check_author_line(fulltext: str, paper_key: str) -> list[str]:
|
|||
|
||||
|
||||
def check_math_spacing(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check for obvious $ ... $ delimiter spacing artifacts."""
|
||||
issues = []
|
||||
for m in MATH_SPACING_ANOMALY.finditer(fulltext):
|
||||
if len(m.group()) > 80:
|
||||
|
|
@ -75,11 +95,10 @@ def check_math_spacing(fulltext: str, paper_key: str) -> list[str]:
|
|||
|
||||
|
||||
def check_glued_math(fulltext: str, paper_key: str) -> list[str]:
|
||||
"""Check for prose/math boundary collapse (Class A)."""
|
||||
issues = []
|
||||
glued_patterns = [
|
||||
re.compile(r'[a-z]\$[0-9\\]'), # text$math
|
||||
re.compile(r'\$[0-9\\][a-z]'), # $math{text
|
||||
re.compile(r'[a-z]\$[0-9\\]'),
|
||||
re.compile(r'\$[0-9\\][a-z]'),
|
||||
]
|
||||
for p in glued_patterns:
|
||||
for m in p.finditer(fulltext):
|
||||
|
|
@ -91,10 +110,8 @@ def check_glued_math(fulltext: str, paper_key: str) -> list[str]:
|
|||
|
||||
|
||||
def check_metadata_authors(meta_path: Path) -> list[str]:
|
||||
"""Check that resolved metadata authors look correct (not biography names)."""
|
||||
if not meta_path.exists():
|
||||
return [" META MISSING: resolved_metadata.json not found"]
|
||||
import json
|
||||
meta = json.loads(meta_path.read_text(encoding='utf-8'))
|
||||
authors = meta.get("authors", meta.get("author", []))
|
||||
if not authors:
|
||||
|
|
@ -109,7 +126,6 @@ def check_metadata_authors(meta_path: Path) -> list[str]:
|
|||
|
||||
|
||||
def verify_paper(key: str, issues: list[str]) -> int:
|
||||
"""Run all checks for one paper. Returns issue count."""
|
||||
paper_root = VAULT / "System" / "PaperForge" / "ocr" / key
|
||||
fulltext_path = paper_root / "fulltext.md"
|
||||
meta_path = paper_root / "metadata" / "resolved_metadata.json"
|
||||
|
|
@ -118,7 +134,6 @@ def verify_paper(key: str, issues: list[str]) -> int:
|
|||
|
||||
count = 0
|
||||
|
||||
# fulltext.md
|
||||
if fulltext_path.exists():
|
||||
text = fulltext_path.read_text(encoding='utf-8', errors='replace')
|
||||
|
||||
|
|
@ -137,18 +152,15 @@ def verify_paper(key: str, issues: list[str]) -> int:
|
|||
issues.append(f" [!] fulltext.md not found for {key}")
|
||||
count += 1
|
||||
|
||||
# resolved_metadata.json
|
||||
meta_issues = check_metadata_authors(meta_path)
|
||||
issues.extend(meta_issues); count += len(meta_issues)
|
||||
|
||||
# Figure notes
|
||||
if figures_dir.exists():
|
||||
for fpath in sorted(figures_dir.glob("*.md")):
|
||||
text = fpath.read_text(encoding='utf-8', errors='replace')
|
||||
ms = check_math_spacing(text, f"{key}/{fpath.name}")
|
||||
issues.extend(ms); count += len(ms)
|
||||
|
||||
# Table notes
|
||||
if tables_dir.exists():
|
||||
for fpath in sorted(tables_dir.glob("*.md")):
|
||||
text = fpath.read_text(encoding='utf-8', errors='replace')
|
||||
|
|
@ -163,23 +175,18 @@ def summary_line(key: str, success: bool, details: str = "") -> str:
|
|||
return f" {icon} {key}{': ' + details if details else ''}"
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
import io
|
||||
|
||||
# Force UTF-8 for stdout to handle Unicode math symbols
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||||
|
||||
print("=" * 70)
|
||||
print(" OCR FINAL POLISH - REBUILD AND VERIFY")
|
||||
print(" OCR REBUILD AND VERIFY")
|
||||
print(f" Vault: {VAULT}")
|
||||
print(f" Keys: {', '.join(KEYS)}")
|
||||
print("=" * 70)
|
||||
|
||||
all_issues = {}
|
||||
all_pass = True
|
||||
|
||||
# Phase 1: Rebuild each paper
|
||||
from paperforge.worker.ocr_rebuild import run_derived_rebuild_for_keys
|
||||
|
||||
for key in KEYS:
|
||||
|
|
@ -187,7 +194,6 @@ if __name__ == "__main__":
|
|||
result = run_derived_rebuild_for_keys(VAULT, [key])
|
||||
print(f" Rebuild result: {json.dumps(result)}")
|
||||
|
||||
# Phase 2: Verify each paper
|
||||
for key in KEYS:
|
||||
print(f"\n--- Verifying {key} ---")
|
||||
issues = []
|
||||
|
|
@ -204,7 +210,6 @@ if __name__ == "__main__":
|
|||
if not paper_ok:
|
||||
all_pass = False
|
||||
|
||||
# Phase 3: Summary
|
||||
print("\n" + "=" * 70)
|
||||
print(" SUMMARY")
|
||||
print("=" * 70)
|
||||
Loading…
Reference in a new issue