From 04c942c8db2371d641cf795dbc0de9be817aff51 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Mon, 8 Jun 2026 12:15:17 +0800 Subject: [PATCH] chore: remove root debug scripts, relocate rebuild_verify to scripts/dev --- _check_code.py | 6 -- _check_ct.py | 25 -------- _dbg_box1.py | 19 ------ _tsckavis_inspect.py | 58 ----------------- .../dev/ocr_rebuild_verify.py | 63 ++++++++++--------- 5 files changed, 34 insertions(+), 137 deletions(-) delete mode 100644 _check_code.py delete mode 100644 _check_ct.py delete mode 100644 _dbg_box1.py delete mode 100644 _tsckavis_inspect.py rename _rebuild_verify.py => scripts/dev/ocr_rebuild_verify.py (82%) diff --git a/_check_code.py b/_check_code.py deleted file mode 100644 index 58d7d7c0..00000000 --- a/_check_code.py +++ /dev/null @@ -1,6 +0,0 @@ -txt = open(r"D:\L\Med\Research\99_System\LiteraturePipeline\github-release\.worktrees\feat-ocr-structured-pipeline\paperforge\worker\ocr_document.py", encoding="utf-8").read() -idx = txt.find('last_insert_anchor_kind == "box"') -if idx >= 0: - print(txt[idx:idx+700]) -else: - print("NOT FOUND") diff --git a/_check_ct.py b/_check_ct.py deleted file mode 100644 index db0c3f9c..00000000 --- a/_check_ct.py +++ /dev/null @@ -1,25 +0,0 @@ -import json, sys -from pathlib import Path -sys.stdout.reconfigure(encoding="utf-8") - -p = Path(r"D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\structure\blocks.structured.jsonl") -rows = [json.loads(line) for line in p.read_text(encoding="utf-8").splitlines() if line.strip()] - -for i in range(123, 132): - r = rows[i] - ctxt = r.get("_container_text") or "" - print(i, r.get("role"), "has_ct=" + str(bool(ctxt)), r.get("bbox"), - (r.get("text") or "")[:100], - "|CT:" + ctxt[:160] if ctxt else "") - -print("---") -ftxt = Path(r"D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\fulltext.md").read_text(encoding="utf-8", errors="replace") -for needle in ["Box 1", "mitochondrial", "Osteoarthritic"]: - idx = ftxt.lower().find(needle.lower()) - if idx != -1: - line_start = ftxt.rfind("\n", 0, idx) + 1 - line_end = ftxt.find("\n", idx) - ctx = ftxt[max(0, idx-40):idx+120] - print(needle, "->", ctx) - else: - print(needle, "-> NOT FOUND") diff --git a/_dbg_box1.py b/_dbg_box1.py deleted file mode 100644 index f17671cd..00000000 --- a/_dbg_box1.py +++ /dev/null @@ -1,19 +0,0 @@ -import json, sys -from pathlib import Path -sys.stdout.reconfigure(encoding='utf-8') - -p = Path(r'D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\structure\blocks.structured.jsonl') -rows = [json.loads(line) for line in p.read_text(encoding='utf-8').splitlines() if line.strip()] - -for i in range(123, 132): - r = rows[i] - print((i, r.get('page'), r.get('role'), r.get('bbox'), (r.get('text') or '')[:140], r.get('_in_visual_container'))) - -print('---') -ftxt = Path(r'D:\L\OB\Literature-hub\System\PaperForge\ocr\TSCKAVIS\fulltext.md').read_text(encoding='utf-8', errors='replace') -for needle in ['In addition to the mitochondrial', 'Osteoarthritic chondrocytes also']: - idx = ftxt.lower().find(needle.lower()) - if idx != -1: - print('FOUND:', needle[:60], 'context:', repr(ftxt[max(0, idx-60):idx+120])) - else: - print('NOT FOUND:', needle[:60]) diff --git a/_tsckavis_inspect.py b/_tsckavis_inspect.py deleted file mode 100644 index 07435e5a..00000000 --- a/_tsckavis_inspect.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -from pathlib import Path - -vault = Path(r"D:\L\OB\Literature-hub") -key = "TSCKAVIS" -root = vault / "System" / "PaperForge" / "ocr" / key -meta_path = root / "metadata" / "resolved_metadata.json" -src_meta_path = root / "metadata" / "source_metadata.json" -struct_path = root / "structure" / "blocks.structured.jsonl" - -if meta_path.exists(): - meta = json.loads(meta_path.read_text(encoding="utf-8")) - print("=== RESOLVED METADATA ===") - for k, v in meta.items(): - print(f" {k}: {json.dumps(v, ensure_ascii=False)[:150]}") -else: - print("METADATA NOT FOUND") - -if src_meta_path.exists(): - src = json.loads(src_meta_path.read_text(encoding="utf-8")) - print("\n=== SOURCE METADATA ===") - print(f' title: {str(src.get("title", ""))[:80]}') - authors = src.get("authors", []) - print(f" authors ({len(authors)}): {authors[:3]}") -else: - print("\nSOURCE METADATA NOT FOUND") - -if struct_path.exists(): - blocks = [json.loads(line) for line in struct_path.read_text(encoding="utf-8").splitlines() if line.strip()] - title_blocks = [b for b in blocks if b.get("role") in ("paper_title", "doc_title")] - author_blocks = [b for b in blocks if b.get("role") == "authors"] - si_blocks = [b for b in blocks if b.get("role") in ("structured_insert", "structured_insert_candidate")] - ni_blocks = [b for b in blocks if b.get("role") == "non_body_insert"] - kp_blocks = [b for b in blocks if "key point" in str(b.get("text") or "").lower()] - print(f"\n=== STRUCTURED BLOCKS ({len(blocks)} total) ===") - print(f" paper_title: {len(title_blocks)}") - for b in title_blocks: - t = (b.get("text") or "")[:80] - print(f" -> {t}") - print(f" authors: {len(author_blocks)}") - for b in author_blocks: - t = (b.get("text") or "")[:80] - print(f" -> {t}") - print(f" structured_insert: {len(si_blocks)}") - for b in si_blocks: - t = (b.get("text") or "")[:80] - print(f" -> {t}") - print(f" non_body_insert: {len(ni_blocks)}") - for b in ni_blocks[:5]: - t = (b.get("text") or "")[:80] - print(f" -> {t}") - if len(ni_blocks) > 5: - print(f" ... ({len(ni_blocks)} total)") - print(f" key-point blocks: {len(kp_blocks)}") - for b in kp_blocks: - r = b.get("role") - t = (b.get("text") or "")[:80] - print(f" role={r} text={t}") diff --git a/_rebuild_verify.py b/scripts/dev/ocr_rebuild_verify.py similarity index 82% rename from _rebuild_verify.py rename to scripts/dev/ocr_rebuild_verify.py index 3d5ddf1c..e88a36ab 100644 --- a/_rebuild_verify.py +++ b/scripts/dev/ocr_rebuild_verify.py @@ -1,17 +1,40 @@ """ -Rebuild and verify Tasks 4-5 of the OCR final polish plan. +Rebuild and verify OCR output quality for specified papers. Usage: - python _rebuild_verify.py + set PAPERFORGE_REAL_OCR_VAULT=D:\L\OB\Literature-hub + set PAPERFORGE_REAL_OCR_KEYS=SAN9AYVR,2GN9LMCW,7C8829BD + python scripts/dev/ocr_rebuild_verify.py -Rebuilds SAN9AYVR, 2GN9LMCW, 7C8829BD and validates output quality. +Environment Variables: + PAPERFORGE_REAL_OCR_VAULT - Path to the Obsidian vault (required) + PAPERFORGE_REAL_OCR_KEYS - Comma-separated paper keys to verify (required) """ -from pathlib import Path +import os import re import sys +import json +import io +from pathlib import Path -VAULT = Path(r"D:\L\OB\Literature-hub") -KEYS = ["SAN9AYVR", "2GN9LMCW", "7C8829BD"] +vault = Path(os.environ.get("PAPERFORGE_REAL_OCR_VAULT", "")) +keys = os.environ.get("PAPERFORGE_REAL_OCR_KEYS", "").split(",") +keys = [k.strip() for k in keys if k.strip()] + +if not vault or not vault.exists(): + print("[!!] PAPERFORGE_REAL_OCR_VAULT is not set or does not exist.") + print(" Set it to a valid Obsidian vault root, e.g.:") + print(' set PAPERFORGE_REAL_OCR_VAULT=D:\\L\\OB\\Literature-hub') + sys.exit(1) + +if not keys: + print("[!!] PAPERFORGE_REAL_OCR_KEYS is not set or empty.") + print(" Set it to comma-separated paper keys, e.g.:") + print(" set PAPERFORGE_REAL_OCR_KEYS=SAN9AYVR,2GN9LMCW,7C8829BD") + sys.exit(1) + +VAULT = vault +KEYS = keys # ---------- helpers ---------- @@ -39,7 +62,6 @@ MATH_SPACING_ANOMALY = re.compile(r'\$\s+[^$]+\s+\$') def check_author_bio_clean(fulltext: str, paper_key: str) -> list[str]: - """Check for residual author-biography prose in body text.""" issues = [] for pattern in BIOGRAPHY_PATTERNS: matches = re.finditer(pattern, fulltext, re.IGNORECASE) @@ -52,7 +74,6 @@ def check_author_bio_clean(fulltext: str, paper_key: str) -> list[str]: def check_author_line(fulltext: str, paper_key: str) -> list[str]: - """Check that author lines don't leak into body.""" issues = [] for pattern in AUTHOR_LINE_PATTERNS: matches = list(re.finditer(pattern, fulltext, re.IGNORECASE)) @@ -62,7 +83,6 @@ def check_author_line(fulltext: str, paper_key: str) -> list[str]: def check_math_spacing(fulltext: str, paper_key: str) -> list[str]: - """Check for obvious $ ... $ delimiter spacing artifacts.""" issues = [] for m in MATH_SPACING_ANOMALY.finditer(fulltext): if len(m.group()) > 80: @@ -75,11 +95,10 @@ def check_math_spacing(fulltext: str, paper_key: str) -> list[str]: def check_glued_math(fulltext: str, paper_key: str) -> list[str]: - """Check for prose/math boundary collapse (Class A).""" issues = [] glued_patterns = [ - re.compile(r'[a-z]\$[0-9\\]'), # text$math - re.compile(r'\$[0-9\\][a-z]'), # $math{text + re.compile(r'[a-z]\$[0-9\\]'), + re.compile(r'\$[0-9\\][a-z]'), ] for p in glued_patterns: for m in p.finditer(fulltext): @@ -91,10 +110,8 @@ def check_glued_math(fulltext: str, paper_key: str) -> list[str]: def check_metadata_authors(meta_path: Path) -> list[str]: - """Check that resolved metadata authors look correct (not biography names).""" if not meta_path.exists(): return [" META MISSING: resolved_metadata.json not found"] - import json meta = json.loads(meta_path.read_text(encoding='utf-8')) authors = meta.get("authors", meta.get("author", [])) if not authors: @@ -109,7 +126,6 @@ def check_metadata_authors(meta_path: Path) -> list[str]: def verify_paper(key: str, issues: list[str]) -> int: - """Run all checks for one paper. Returns issue count.""" paper_root = VAULT / "System" / "PaperForge" / "ocr" / key fulltext_path = paper_root / "fulltext.md" meta_path = paper_root / "metadata" / "resolved_metadata.json" @@ -118,7 +134,6 @@ def verify_paper(key: str, issues: list[str]) -> int: count = 0 - # fulltext.md if fulltext_path.exists(): text = fulltext_path.read_text(encoding='utf-8', errors='replace') @@ -137,18 +152,15 @@ def verify_paper(key: str, issues: list[str]) -> int: issues.append(f" [!] fulltext.md not found for {key}") count += 1 - # resolved_metadata.json meta_issues = check_metadata_authors(meta_path) issues.extend(meta_issues); count += len(meta_issues) - # Figure notes if figures_dir.exists(): for fpath in sorted(figures_dir.glob("*.md")): text = fpath.read_text(encoding='utf-8', errors='replace') ms = check_math_spacing(text, f"{key}/{fpath.name}") issues.extend(ms); count += len(ms) - # Table notes if tables_dir.exists(): for fpath in sorted(tables_dir.glob("*.md")): text = fpath.read_text(encoding='utf-8', errors='replace') @@ -163,23 +175,18 @@ def summary_line(key: str, success: bool, details: str = "") -> str: return f" {icon} {key}{': ' + details if details else ''}" -# ---------- main ---------- - if __name__ == "__main__": - import json - import io - - # Force UTF-8 for stdout to handle Unicode math symbols sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') print("=" * 70) - print(" OCR FINAL POLISH - REBUILD AND VERIFY") + print(" OCR REBUILD AND VERIFY") + print(f" Vault: {VAULT}") + print(f" Keys: {', '.join(KEYS)}") print("=" * 70) all_issues = {} all_pass = True - # Phase 1: Rebuild each paper from paperforge.worker.ocr_rebuild import run_derived_rebuild_for_keys for key in KEYS: @@ -187,7 +194,6 @@ if __name__ == "__main__": result = run_derived_rebuild_for_keys(VAULT, [key]) print(f" Rebuild result: {json.dumps(result)}") - # Phase 2: Verify each paper for key in KEYS: print(f"\n--- Verifying {key} ---") issues = [] @@ -204,7 +210,6 @@ if __name__ == "__main__": if not paper_ok: all_pass = False - # Phase 3: Summary print("\n" + "=" * 70) print(" SUMMARY") print("=" * 70)