obsidian-plugin/lllin000_PaperForge
1
0
Fork 0
mirror of https://github.com/lllin000/PaperForge.git synced 2026-07-22 06:50:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add streaming OCR rebuild progress

Browse source
This commit is contained in:
LLLin000 2026-07-14 18:05:49 +08:00
parent 3a0d171e31
commit 349d4f6dcb
11 changed files with 4450 additions and 2714 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

213
paperforge/commands/ocr.py
View file

@ -2,6 +2,8 @@
import argparse
import json
import signal
from collections.abc import Callable
import logging
from pathlib import Path
@ -225,19 +227,77 @@ def _get_run_ocr():
return run_ocr
def _make_cooperative_stop() -> tuple[Callable[[], bool], Callable[[], None]]:
"""Install cooperative stop mechanisms: SIGINT handler + stdin reader.
SIGINT (POSIX/terminal): sets flag.
Stdin line "PAPERFORGE_STOP": daemon reader thread sets flag
(reliable on Windows where SIGINT can't be caught in subprocesses).
Returns (is_stopped, restore):
is_stopped(): returns True if stop was requested
restore(): restores original SIGINT handler
"""
import threading as _threading
_flag: list[bool] = [False]
_reader_active: list[bool] = [True]
# ── SIGINT handler (POSIX/terminal) ──
def _handler(_signum: int, _frame: object) -> None:
_flag[0] = True
_old = signal.signal(signal.SIGINT, _handler)
# ── Stdin reader daemon thread (Windows-reliable) ──
def _stdin_reader() -> None:
import sys as _sys
try:
while _reader_active[0]:
line = _sys.stdin.readline()
if not line: # EOF (pipe closed)
break
if line.strip() == "PAPERFORGE_STOP":
_flag[0] = True
break
except (OSError, ValueError, AttributeError, RuntimeError):
pass # stdin unavailable in testing/headless
_reader_thread = _threading.Thread(target=_stdin_reader, daemon=True)
_reader_thread.start()
def _restore() -> None:
_reader_active[0] = False
signal.signal(signal.SIGINT, _old)
return (lambda: _flag[0], _restore)
def _run_ocr_redo(vault: Path, keys: list[str] | None = None, dry_run: bool = False,
verbose: bool = False, no_progress: bool = False) -> int:
"""Re-run OCR for papers.
If keys provided, only redo those papers.
If no keys, scan for ocr_redo: true papers (old behavior).
"""
from paperforge.worker.ocr import ocr_redo_papers
if keys:
# Keyed redo: process specific papers
from paperforge.worker.ocr import ensure_ocr_meta, write_json
If keys provided, delegate to redo_papers_for_keys which handles the
full artifact-delete + OCR run + post-check cycle per paper, supporting
per-paper progress tokens and cooperative stop.
If no keys, scan for ocr_redo: true papers (legacy behavior).
Progress tokens (multi-key non-dry-run only):
OCR_REDO_START:{total}
OCR_REDO_PROGRESS:{current}:{total}:{key}
OCR_REDO_DONE
Cooperative stop: SIGINT sets flag checked between papers.
"""
from paperforge.worker.ocr import ocr_redo_papers, redo_papers_for_keys
if not keys:
return ocr_redo_papers(vault, dry_run=dry_run, verbose=verbose, no_progress=no_progress)
if not dry_run:
from paperforge.worker._utils import pipeline_paths
from pathlib import Path
paths = pipeline_paths(vault)
ocr_root = paths.get("ocr")
@ -245,34 +305,65 @@ def _run_ocr_redo(vault: Path, keys: list[str] | None = None, dry_run: bool = Fa
print("No OCR root directory.")
return 1
for key in keys:
meta_path = ocr_root / key / "meta.json"
if not meta_path.exists():
print(f"{key}: no meta.json, skipping")
continue
meta = __import__("json").loads(meta_path.read_text(encoding="utf-8"))
current = str(meta.get("ocr_status", "") or "").strip().lower()
if current == "nopdf":
print(f"{key}: nopdf — cannot redo (missing PDF)")
continue
# Reset meta to pending
meta["ocr_status"] = "pending"
meta["ocr_job_id"] = ""
meta["ocr_started_at"] = ""
meta["ocr_finished_at"] = ""
meta["error"] = ""
meta["retry_count"] = 0
if dry_run:
print(f"{key}: would reset to pending (dry-run)")
else:
write_json(meta_path, meta)
print(f"{key}: reset to pending")
total = len(keys)
batch = total > 1 and not dry_run
if not dry_run:
print(f"Triggering OCR run for {len(keys)} paper(s)...")
if batch:
print(f"OCR_REDO_START:{total}", flush=True)
_is_stopped, _restore_signal = _make_cooperative_stop()
else:
_is_stopped = lambda: False
_restore_signal = lambda: None
if dry_run:
print(f"Would redo {total} paper(s):")
for k in keys:
print(f" - {k}: would delete artifacts and re-run OCR")
print("Dry-run: no changes made. Run without --dry-run to execute.")
if batch:
print(f"OCR_REDO_DONE", flush=True)
return 0
return ocr_redo_papers(vault, dry_run=dry_run, verbose=verbose, no_progress=no_progress)
# Track current for progress token
_current = [0]
def _progress_callback(key: str) -> None:
_current[0] += 1
print(f"OCR_REDO_PROGRESS:{_current[0]}:{total}:{key}", flush=True)
def _stop_check() -> bool:
return _is_stopped()
try:
result = redo_papers_for_keys(
vault, keys,
verbose=verbose,
progress_callback=_progress_callback if batch else None,
stop_check=_stop_check if batch else None,
)
success_keys = result.get("success_keys", [])
failed_keys = result.get("failed_keys", [])
worker_exit_code = result.get("exit_code", 0)
if success_keys:
print(f"Redo OCR done={len(success_keys)}: {', '.join(success_keys)}", flush=True)
if failed_keys:
print(f"Redo OCR pending/failed={len(failed_keys)}: {', '.join(failed_keys)}", flush=True)
if batch and _is_stopped():
print(f"Batch stopped (SIGINT) after {_current[0]} paper(s).")
if batch:
print(f"OCR_REDO_DONE", flush=True)
finally:
_restore_signal()
if batch and _is_stopped():
return 130
return worker_exit_code
def _run_ocr_list(vault: Path, json_output: bool = False, output_file: str | None = None,
@ -438,7 +529,13 @@ def _run_ocr_rebuild(
resume: bool = False,
parallel_workers: int = 4,
) -> int:
"""Rebuild OCR-derived artifacts from existing raw blocks."""
"""Rebuild OCR-derived artifacts from existing raw blocks.
Progress tokens (multi-key non-dry-run only):
OCR_REBUILD_START:{total}
OCR_REBUILD_PROGRESS:{current}:{total}:{key}
OCR_REBUILD_DONE
"""
from paperforge.worker.ocr_maintenance import collect_maintenance_rows
from paperforge.worker.ocr_rebuild import run_derived_rebuild_for_keys
@ -452,22 +549,52 @@ def _run_ocr_rebuild(
if resume:
print("Note: OCR rebuild resume is now version/artifact based; .done markers are ignored.")
total = len(selected)
batch = total > 1 and not dry_run
if dry_run:
print(f"Would rebuild {len(selected)} paper(s):")
print(f"Would rebuild {total} paper(s):")
for k in selected:
reason = reasons.get(k, "manual_override")
print(f" - {k}: {reason}")
return 0
from paperforge.worker._progress import progress_bar
result = run_derived_rebuild_for_keys(
vault, selected,
progress_bar=progress_bar,
parallel=parallel_workers,
)
count = result.get("rebuild_count", 0)
print(f"Done. Rebuilt {count} paper(s).")
return 0
if batch:
print(f"OCR_REBUILD_START:{total}", flush=True)
_count = 0
def _on_progress(key: str) -> None:
nonlocal _count
_count += 1
print(f"OCR_REBUILD_PROGRESS:{_count}:{total}:{key}", flush=True)
# Force sequential for cooperative stop; parallel pool can't stop mid-batch
parallel_workers = 0
_is_stopped, _restore_signal = _make_cooperative_stop()
def _stop_check() -> bool:
return _is_stopped()
else:
_on_progress = None # type: ignore[assignment]
_is_stopped = lambda: False
_restore_signal = lambda: None
_stop_check = None
try:
result = run_derived_rebuild_for_keys(
vault, selected,
progress_bar=progress_bar,
parallel=parallel_workers,
on_progress=_on_progress,
stop_check=_stop_check,
)
count = result.get("rebuild_count", 0)
print(f"Done. Rebuilt {count} paper(s).")
if batch:
print(f"OCR_REBUILD_DONE", flush=True)
finally:
_restore_signal()
return 0 if not _is_stopped() else 130
def run(args: argparse.Namespace) -> int:

4826
paperforge/plugin/main.js
View file

File diff suppressed because it is too large Load diff

15
paperforge/plugin/src/i18n.ts
View file

@ -268,6 +268,13 @@ const LANG: Record<string, Record<string, string>> = {
maintenance_refresh_spinning: "Updating…",
maintenance_all_good: "✅ All good — no action needed",
maintenance_n_pending: "{n} need attention",
maintenance_filter_all: "All",
maintenance_filter_recommended: "Recommended",
maintenance_batch_rebuild: "▶ Rebuild selected",
maintenance_batch_redo: "▶ Full OCR redo selected",
maintenance_stop: "Stop",
maintenance_batch_complete: "Batch operation complete — {n} papers processed.",
maintenance_progress_label: "{current}/{total} papers",
version_panel_title: "Version History",
version_panel_back: "Back",
version_filter_placeholder: "Filter papers...",
@ -559,6 +566,7 @@ const LANG: Record<string, Record<string, string>> = {
"这类论文通常表示版式复杂或信号偏弱，PaperForge 目前没有高置信度的维护建议。",
ocr_maint_all_papers: "全部论文",
ocr_maint_rebuild_btn: "重建结果",
ocr_maint_redo_btn: "重新 OCR",
maintenance_group_retry: "需要重试",
maintenance_group_rebuild: "可重建结果",
maintenance_group_legacy: "可升级旧结果（可选）",
@ -568,6 +576,13 @@ const LANG: Record<string, Record<string, string>> = {
maintenance_refresh_spinning: "正在更新…",
maintenance_all_good: "✅ 全部正常",
maintenance_n_pending: "{n} 篇需要处理",
maintenance_filter_all: "全部",
maintenance_filter_recommended: "建议处理",
maintenance_batch_rebuild: "▶ 重建已选",
maintenance_batch_redo: "▶ 全部重跑 OCR",
maintenance_stop: "停止",
maintenance_batch_complete: "批量操作完成 — 处理了 {n} 篇论文。",
maintenance_progress_label: "{current}/{total} 篇",
version_panel_title: "版本历史",
version_panel_back: "返回",
version_filter_placeholder: "搜索论文...",

109
paperforge/plugin/src/services/progress-parser.ts Normal file
View file

@ -0,0 +1,109 @@
/**
* Shared progress token parser — handles EMBED, OCR_REBUILD, and OCR_REDO
* tokens across arbitrary stdout chunks.
*
* Token formats:
* {PREFIX}_START:{total}
* {PREFIX}_PROGRESS:{current}:{total}:{key}
* {PREFIX}_DONE
*
* PREFIX in: EMBED, OCR_REBUILD, OCR_REDO
*
* This is the stable plugin boundary contract from the CLI backend.
*/
export type ProgressEventType = "START" | "PROGRESS" | "DONE";
export interface ProgressEvent {
prefix: string;
event: ProgressEventType;
total?: number;
current?: number;
key?: string;
}
const KNOWN_PREFIXES = ["EMBED", "OCR_REBUILD", "OCR_REDO"];
/**
* Parse an arbitrarily chunked stdout stream for progress tokens.
*
* @param chunk Raw text from the latest stdout data event.
* @param buffer Accumulated incomplete line from a previous call (empty string initially).
* @returns Parsed events and any leftover line fragment to pass to the next call.
*/
export function processProgressChunk(
chunk: string,
buffer: string,
): { events: ProgressEvent[]; buffer: string } {
const full = buffer + chunk;
const lines = full.split("\n");
// The last element may be an incomplete line — hold for next chunk.
const incomplete = lines.pop() ?? "";
const events: ProgressEvent[] = [];
for (const line of lines) {
for (const prefix of KNOWN_PREFIXES) {
const pLen = prefix.length;
if (line.startsWith(prefix + "_START:")) {
const total = parseInt(line.slice(pLen + 7) /* "_START:".length */, 10) || 0;
events.push({ prefix, event: "START", total });
break;
}
if (line.startsWith(prefix + "_PROGRESS:")) {
const rest = line.slice(pLen + 10); /* "_PROGRESS:".length */
const parts = rest.split(":");
events.push({
prefix,
event: "PROGRESS",
current: parseInt(parts[0], 10) || 0,
total: parseInt(parts[1], 10) || 0,
key: parts[2] ?? "",
});
break;
}
if (line === prefix + "_DONE" || line.startsWith(prefix + "_DONE:")) {
events.push({ prefix, event: "DONE" });
break;
}
}
}
return { events, buffer: incomplete };
}
/**
* Parse a single complete line for a progress token.
* Useful when text is already newline-aligned (e.g. execFile stdout).
*/
export function parseProgressLine(line: string): ProgressEvent | null {
for (const prefix of KNOWN_PREFIXES) {
const pLen = prefix.length;
if (line.startsWith(prefix + "_START:")) {
const total = parseInt(line.slice(pLen + 7), 10) || 0;
return { prefix, event: "START", total };
}
if (line.startsWith(prefix + "_PROGRESS:")) {
const rest = line.slice(pLen + 10);
const parts = rest.split(":");
return {
prefix,
event: "PROGRESS",
current: parseInt(parts[0], 10) || 0,
total: parseInt(parts[1], 10) || 0,
key: parts[2] ?? "",
};
}
if (line === prefix + "_DONE" || line.startsWith(prefix + "_DONE:")) {
return { prefix, event: "DONE" };
}
}
return null;
}

733
paperforge/plugin/src/settings.ts
View file

@ -40,6 +40,8 @@ import {
writeMaintenanceCache,
refreshMaintenanceData,
} from "./services/ocr-maintenance-ui";
import { processProgressChunk } from "./services/progress-parser";
// ── Interface ──
@ -56,6 +58,11 @@ interface ISettingPlugin {
_embedProcess?: unknown;
_embedProgress?: { current: number; total: number; key: string };
_embedStderr?: string;
_embedBuffer?: string;
_ocrProcess?: unknown;
_ocrProgress?: { current: number; total: number; key: string };
_ocrBuffer?: string;
_ocrWasStopped?: boolean;
_embedPollInterval?: ReturnType<typeof setInterval> | null;
_embedPolling?: boolean;
}
@ -1260,16 +1267,19 @@ export class PaperForgeSettingTab extends PluginSettingTab {
: Buffer.isBuffer(data)
? data.toString("utf-8")
: String(data);
const lines = text.split("\n");
for (const line of lines) {
if (line.startsWith("EMBED_START:")) {
this.plugin._embedProgress!.total =
parseInt(line.split(":")[1]) || 0;
} else if (line.startsWith("EMBED_PROGRESS:")) {
const parts = line.split(":");
this.plugin._embedProgress!.current = parseInt(parts[1]) || 0;
this.plugin._embedProgress!.key = parts[3] || "";
} else if (line.startsWith("EMBED_DONE")) {
// Use shared parser — inline buffer reset on each build
const { events, buffer } = processProgressChunk(
text,
this.plugin._embedBuffer ?? "",
);
this.plugin._embedBuffer = buffer;
for (const ev of events) {
if (ev.event === "START") {
this.plugin._embedProgress!.total = ev.total || 0;
} else if (ev.event === "PROGRESS") {
this.plugin._embedProgress!.current = ev.current || 0;
this.plugin._embedProgress!.key = ev.key || "";
} else if (ev.event === "DONE") {
this.plugin._embedProcess = null;
this.plugin._embedProgress!.current =
this.plugin._embedProgress!.total;
@ -1956,23 +1966,14 @@ export class PaperForgeSettingTab extends PluginSettingTab {
text: t("tab_maintenance") || "维护",
});
const vaultPath = (this.app.vault.adapter as any).basePath as string;
// vault path — DataAdapter.basePath is undocumented but stable
const adapter = this.app.vault
.adapter as unknown as { basePath?: string };
const vaultPath = adapter.basePath ?? "";
const statusEl = containerEl.createEl("div");
// ── Helpers ──
const buildActionArgs = (keys: string[], action: string) => {
if (action === "retry_ocr" || action === "upgrade_legacy")
return {
cmd: ["-m", "paperforge", "ocr", "redo", ...keys],
timeout: 300000,
};
if (action === "rebuild_result")
return {
cmd: ["-m", "paperforge", "ocr", "rebuild", ...keys],
timeout: 120000,
};
return null;
};
// Filter state
const filterState = { active: "all" as "all" | "recommended" };
// ── Phase 1: Read cache ──
let cache: MaintenanceCache | null = null;
@ -1983,7 +1984,8 @@ export class PaperForgeSettingTab extends PluginSettingTab {
// ── Phase 2: Try manifest refresh ──
const py = resolvePythonExecutable(
vaultPath,
this.plugin.settings as any,
// PaperForgeSettings — no cast needed, ISettingPlugin has it
this.plugin.settings,
fs,
execFileSync
);
@ -1995,223 +1997,475 @@ export class PaperForgeSettingTab extends PluginSettingTab {
return;
}
// Shared table rendering function
const renderTable = (papers: MaintenanceDisplayRow[]) => {
statusEl.empty();
const isBatchRunning = () => !!this.plugin._ocrProcess;
const visible = papers.filter((p) => p.visible_in_maintenance);
const renderTable = (papers: MaintenanceDisplayRow[]) => {
statusEl.empty();
const allVisible = papers.filter((p) => p.visible_in_maintenance);
// Filter tabs — render before empty check so user can always switch back
const filterRow = statusEl.createEl("div", {
cls: "pf-maint-filters",
});
const allTab = filterRow.createEl("button", {
cls:
"pf-maint-filter" +
(filterState.active === "all" ? " active" : ""),
text: t("maintenance_filter_all") || "All",
});
allTab.addEventListener("click", () => {
filterState.active = "all";
renderTable(papers);
});
const recTab = filterRow.createEl("button", {
cls:
"pf-maint-filter" +
(filterState.active === "recommended" ? " active" : ""),
text: t("maintenance_filter_recommended") || "Recommended",
});
recTab.addEventListener("click", () => {
filterState.active = "recommended";
renderTable(papers);
});
// Recommended = items with rebuild display_group or rebuild_result display_action
const visible =
filterState.active === "recommended"
? allVisible.filter(
(p) =>
p.display_group === "rebuild" ||
p.display_action === "rebuild_result",
)
: allVisible;
// If the active filter yields nothing, show a message and skip the table/progress
if (visible.length === 0) {
statusEl.createEl("p", {
text: t("maintenance_all_good") || "✅ 全部正常",
text: "当前筛选条件下无数据",
cls: "setting-item-description",
});
return;
}
} else {
const pyPath = py.path;
const pyExtra = (py.extraArgs || []) as string[];
const pyPath = py.path;
const pyExtra = (py.extraArgs || []) as string[];
// Group by display_group
const groups: {
key: string;
title: string;
items: MaintenanceDisplayRow[];
}[] = [
{
key: "retry",
title: t("maintenance_group_retry") || "需要重试",
items: [],
},
{
key: "rebuild",
title: t("maintenance_group_rebuild") || "可重建结果",
items: [],
},
{
key: "legacy_optional",
title: t("maintenance_group_legacy") || "可升级旧结果（可选）",
items: [],
},
];
for (const p of visible) {
const g = groups.find((g) => g.key === p.display_group);
if (g) g.items.push(p);
}
// ── Progress bar (if batch running) — mutable DOM refs, no full re-render ──
const progressContainer = statusEl.createEl("div", {
cls: "pf-maint-progress",
});
progressContainer.style.display = "none";
for (const group of groups) {
if (group.items.length === 0) continue;
const track = progressContainer.createEl("div", {
cls: "paperforge-progress-track",
});
track.style.cssText = "flex:1;";
const doneSeg = track.createEl("div", {
cls: "paperforge-progress-seg done",
});
const pendingSeg = track.createEl("div", {
cls: "paperforge-progress-seg pending",
});
const label = progressContainer.createEl("span", {
cls: "pf-maint-progress-text",
});
const keyLabel = progressContainer.createEl("span", {
cls: "pf-maint-progress-key",
});
const isLegacy = group.key === "legacy_optional";
const section = isLegacy
? statusEl.createEl("details")
: statusEl.createEl("div");
// Stop button — cooperative: stdin control line, fallback to SIGINT
const stopBtn = progressContainer.createEl("button", {
text: t("maintenance_stop") || "Stop",
});
stopBtn.className = "mod-warning";
stopBtn.addEventListener("click", () => {
const child = this.plugin._ocrProcess as unknown as {
stdin?: { write: (_: string) => boolean };
kill?: (_: string) => void;
};
if (child) {
// Prefer stdin control line (cooperative — backend finishes current paper)
if (child.stdin && typeof child.stdin.write === "function") {
child.stdin.write("PAPERFORGE_STOP\n");
} else if (typeof child.kill === "function") {
child.kill("SIGINT");
}
}
// Flag, don't null — onClose handles cleanup
this.plugin._ocrWasStopped = true;
stopBtn.disabled = true;
stopBtn.textContent = (t("maintenance_stop") || "Stop") + "…";
});
if (isLegacy) {
const summary = (section as HTMLDetailsElement).createEl("summary");
summary.createEl("strong", {
text: group.title + " (" + group.items.length + ")",
});
// In-place DOM update — called from runBatch start + onData events
const updateProgress = () => {
const prog = this.plugin._ocrProgress;
if (!prog || prog.total === 0 || !this.plugin._ocrProcess) {
progressContainer.style.display = "none";
return;
}
progressContainer.style.display = "flex";
const pct =
prog.total > 0
? ((prog.current / prog.total) * 100).toFixed(1)
: "0";
doneSeg.style.width = `${pct}%`;
doneSeg.style.minWidth = prog.current > 0 ? "2px" : "0";
if (prog.current < prog.total) {
pendingSeg.style.display = "";
pendingSeg.style.flex = "1";
} else {
section.createEl("h3", {
text: group.title + " (" + group.items.length + ")",
});
pendingSeg.style.display = "none";
}
// Selection state per row
const selState = new Map<string, boolean>();
for (const p of group.items) selState.set(p.key, false);
label.textContent = (
t("maintenance_progress_label") ||
"{current}/{total} papers"
)
.replace("{current}", String(prog.current))
.replace("{total}", String(prog.total));
// Toolbar
const toolbar = section.createEl("div", { cls: "pf-maint-toolbar" });
const selAllBtn = toolbar.createEl("button", { text: "全选" });
const deselAllBtn = toolbar.createEl("button", { text: "取消全选" });
const execBtn = toolbar.createEl("button", {
text: "▶ 执行已选",
cls: "mod-cta",
keyLabel.textContent = prog.key ? ` (${prog.key})` : "";
};
updateProgress();
// Selection state
const selState = new Map<string, boolean>();
for (const p of visible) selState.set(p.key, false);
// ── Table ──
const wrapper = statusEl.createEl("div", {
cls: "pf-maint-table-wrap",
});
const table = wrapper.createEl("table", { cls: "pf-maint-table" });
const thead = table.createEl("thead");
const tbody = table.createEl("tbody");
const headerRow = thead.insertRow();
["", "Paper", "Status Reason", "Actions"].forEach((h) => {
const th = document.createElement("th");
th.textContent = h;
headerRow.appendChild(th);
});
const isBusy = isBatchRunning();
for (const p of visible) {
const tr = tbody.insertRow();
// Checkbox
const selTd = tr.insertCell();
selTd.style.cssText =
"padding:3px 4px;text-align:center;width:24px;";
const cb = document.createElement("input");
cb.type = "checkbox";
cb.className = "pf-maint-sel";
cb.checked = selState.get(p.key) || false;
cb.addEventListener("change", () => {
selState.set(p.key, cb.checked);
updateBatchLabel();
});
const execLabel = toolbar.createEl("span", {
cls: "pf-maint-exec-label",
selTd.appendChild(cb);
// Paper info (title + key)
const infoTd = tr.insertCell();
infoTd.style.cssText = "padding:3px 4px;";
const infoDiv = infoTd.createEl("div", {
cls: "pf-maint-paper-info",
});
infoDiv.createEl("div", {
cls: "pf-maint-paper-title",
text: p.title || p.key,
});
infoDiv.createEl("div", {
cls: "pf-maint-paper-key",
text: p.key,
});
const updateExecLabel = () => {
const n = group.items.filter((p) => selState.get(p.key)).length;
execLabel.setText("已选 " + n + " 篇");
};
updateExecLabel();
// Status reason
const reasonTd = tr.insertCell();
reasonTd.style.cssText = "padding:3px 4px;";
reasonTd.createEl("div", {
cls: "pf-maint-reason",
text: p.display_reason || "",
});
selAllBtn.addEventListener("click", () => {
for (const p of group.items) selState.set(p.key, true);
updateExecLabel();
// Update checkboxes
const cbs = section.querySelectorAll(
"input[type=checkbox].pf-maint-sel"
);
cbs.forEach((cb) => {
(cb as HTMLInputElement).checked = true;
// Action buttons — [Rebuild] [Redo]
const actionTd = tr.insertCell();
actionTd.style.cssText =
"padding:3px 4px;white-space:nowrap;";
const actionDiv = actionTd.createEl("div", {
cls: "pf-maint-actions",
});
if (p.can_rebuild) {
const rebuildBtn = actionDiv.createEl("button", {
cls: "pf-maint-action-btn rebuild",
text: t("maintenance_btn_rebuild") || "Rebuild",
});
});
deselAllBtn.addEventListener("click", () => {
for (const p of group.items) selState.set(p.key, false);
updateExecLabel();
const cbs = section.querySelectorAll(
"input[type=checkbox].pf-maint-sel"
);
cbs.forEach((cb) => {
(cb as HTMLInputElement).checked = false;
});
});
execBtn.addEventListener("click", () => {
const selected = group.items.filter((p) => selState.get(p.key));
if (selected.length === 0) {
new Notice("请先选择要处理的论文。");
return;
}
for (const p of selected) {
const args = buildActionArgs([p.key], p.display_action);
if (!args) continue;
if (isBusy) rebuildBtn.disabled = true;
rebuildBtn.addEventListener("click", () => {
const args = [
...pyExtra,
"-m",
"paperforge",
"ocr",
"rebuild",
p.key,
];
execFile(
pyPath,
[...pyExtra, ...args.cmd],
{ cwd: vaultPath, timeout: args.timeout, windowsHide: true },
args,
{
cwd: vaultPath,
timeout: 120000,
windowsHide: true,
},
() => {
new Notice(p.display_label + " — " + p.key);
new Notice(
(t("maintenance_btn_rebuild") || "Rebuild") +
" — " +
p.key
);
}
);
}
});
// Table
const wrapper = section.createEl("div", { cls: "pf-maint-table-wrap" });
const table = wrapper.createEl("table", { cls: "pf-maint-table" });
const thead = table.createEl("thead");
const tbody = table.createEl("tbody");
const headerRow = thead.insertRow();
["", "Key", "Title", "建议操作", "原因", "操作"].forEach((h) => {
const th = document.createElement("th");
th.textContent = h;
headerRow.appendChild(th);
});
const btnText = (action: string) => {
if (action === "retry_ocr")
return t("maintenance_btn_retry") || "重试";
if (action === "rebuild_result")
return t("maintenance_btn_rebuild") || "重建";
if (action === "upgrade_legacy")
return t("maintenance_btn_upgrade") || "升级";
return "";
};
for (const p of group.items) {
const tr = tbody.insertRow();
// Checkbox
const selTd = tr.insertCell();
selTd.style.cssText = "padding:3px 4px;text-align:center;";
const cb = document.createElement("input");
cb.type = "checkbox";
cb.className = "pf-maint-sel";
cb.checked = selState.get(p.key) || false;
cb.addEventListener("change", () => {
selState.set(p.key, cb.checked);
updateExecLabel();
});
selTd.appendChild(cb);
}
// Key
const keyTd = tr.insertCell();
keyTd.style.cssText =
"padding:3px 4px;white-space:nowrap;font-size:11px;max-width:90px;overflow:hidden;text-overflow:ellipsis;";
keyTd.textContent = p.key;
// Title
const titleTd = tr.insertCell();
titleTd.style.cssText =
"padding:3px 4px;white-space:nowrap;max-width:220px;overflow:hidden;text-overflow:ellipsis;";
titleTd.textContent = p.title || p.key;
// Display label (建议操作)
const labelTd = tr.insertCell();
labelTd.style.cssText = "padding:3px 4px;white-space:nowrap;";
labelTd.textContent = p.display_label;
// Reason
const reasonTd = tr.insertCell();
reasonTd.style.cssText =
"padding:3px 4px;white-space:nowrap;max-width:160px;overflow:hidden;text-overflow:ellipsis;font-size:11px;color:var(--text-muted);";
reasonTd.textContent = p.display_reason || "";
// Action button
const actionTd = tr.insertCell();
actionTd.style.cssText =
"padding:3px 4px;text-align:center;white-space:nowrap;";
const actionBtn = document.createElement("button");
actionBtn.textContent = btnText(p.display_action);
if (actionBtn.textContent) {
actionBtn.addEventListener("click", () => {
const args = buildActionArgs([p.key], p.display_action);
if (!args) return;
execFile(
pyPath,
[...pyExtra, ...args.cmd],
{ cwd: vaultPath, timeout: args.timeout, windowsHide: true },
() => {
new Notice(p.display_label + " — " + p.key);
}
);
});
actionTd.appendChild(actionBtn);
}
if (p.can_redo) {
const redoBtn = actionDiv.createEl("button", {
cls: "pf-maint-action-btn redo",
text: t("ocr_maint_redo_btn") || "Redo",
});
if (isBusy) redoBtn.disabled = true;
redoBtn.addEventListener("click", () => {
const args = [
...pyExtra,
"-m",
"paperforge",
"ocr",
"redo",
p.key,
];
execFile(
pyPath,
args,
{
cwd: vaultPath,
timeout: 300000,
windowsHide: true,
},
() => {
new Notice(
(t("ocr_maint_redo_btn") || "Redo OCR") +
" — " +
p.key
);
}
);
});
}
}
// ── Batch action bar ──
const batchBar = statusEl.createEl("div", {
cls: "pf-maint-batch-bar",
});
const batchLabel = batchBar.createEl("span", {
cls: "pf-maint-batch-label",
text: "0 selected",
});
const updateBatchLabel = () => {
const n = visible.filter((p) => selState.get(p.key)).length;
batchLabel.textContent = n + " selected";
};
const rebuildBatchBtn = batchBar.createEl("button", {
cls: "mod-cta",
text:
t("maintenance_batch_rebuild") || "▶ Rebuild selected",
});
rebuildBatchBtn.disabled = isBusy;
const redoBatchBtn = batchBar.createEl("button", {
cls: "mod-cta",
text:
t("maintenance_batch_redo") || "▶ Full OCR redo selected",
});
redoBatchBtn.disabled = isBusy;
const runBatch = (action: "rebuild" | "redo") => {
const selected = visible.filter((p) =>
selState.get(p.key)
);
if (selected.length === 0) {
new Notice("Please select papers first.");
return;
}
const keys = selected.map((p) => p.key);
this.plugin._ocrProgress = {
current: 0,
total: keys.length,
key: "",
};
this.plugin._ocrBuffer = "";
this.plugin._ocrWasStopped = false;
const prefix =
action === "rebuild" ? "OCR_REBUILD" : "OCR_REDO";
// Disable batch + per-row action buttons during the batch
rebuildBatchBtn.disabled = true;
redoBatchBtn.disabled = true;
Array.from(
wrapper.querySelectorAll<HTMLButtonElement>(".pf-maint-action-btn"),
).forEach((btn) => {
btn.disabled = true;
});
// Also disable checkboxes to prevent selection changes during batch
Array.from(
wrapper.querySelectorAll<HTMLInputElement>(".pf-maint-sel"),
).forEach((cb) => {
cb.disabled = true;
});
// Disable filter tabs to prevent filter switch mid-batch
allTab.disabled = true;
recTab.disabled = true;
stopBtn.disabled = false;
stopBtn.textContent = t("maintenance_stop") || "Stop";
const child = this._callPython(
["ocr", action, ...keys],
{
stream: true,
onData: (data: unknown) => {
const text =
typeof data === "string"
? data
: Buffer.isBuffer(data)
? data.toString("utf-8")
: String(data);
// Shared parser with chunk buffer
const { events, buffer } = processProgressChunk(
text,
this.plugin._ocrBuffer ?? "",
);
this.plugin._ocrBuffer = buffer;
for (const ev of events) {
if (ev.event === "START") {
if (this.plugin._ocrProgress) {
this.plugin._ocrProgress.total =
ev.total || keys.length;
}
} else if (ev.event === "PROGRESS") {
this.plugin._ocrProgress = {
current: ev.current || 0,
total: ev.total || keys.length,
key: ev.key || "",
};
}
// DONE handled in onClose
}
// In-place DOM update — no full re-render
updateProgress();
},
onError: (err: Error) => {
this.plugin._ocrProcess = null;
new Notice(
"Batch error: " + (err.message || err),
);
renderTable(papers);
},
onClose: (code: number | null) => {
// code 130 = SIGINT caught cooperatively by the backend
if (this.plugin._ocrWasStopped || code === 130) {
this.plugin._ocrWasStopped = false;
// Leave progress as-is; no finalize
this.plugin._ocrProcess = null;
updateProgress();
new Notice("OCR batch stopped by user.");
} else if (code === 0) {
// Finalize progress to show completion even if no tokens came through
if (this.plugin._ocrProgress) {
this.plugin._ocrProgress.current =
this.plugin._ocrProgress.total;
}
this.plugin._ocrProcess = null;
updateProgress();
new Notice(
(
t("maintenance_batch_complete") ||
"Batch operation complete — {n} papers processed."
).replace("{n}", String(keys.length)),
);
} else {
this.plugin._ocrProcess = null;
updateProgress();
new Notice(
"Batch operation finished with exit code " +
code +
".",
8000,
);
}
// Full refresh
refreshMaintenanceData(
vaultPath,
pyPath,
pyExtra,
cache,
)
.then((result) => {
if (result.changed || !cache) {
cache = {
manifest: {},
papers: Object.fromEntries(
result.data.map(
(p: MaintenanceDisplayRow) => [p.key, p],
),
),
cached_at: new Date().toISOString(),
};
writeMaintenanceCache(vaultPath, cache);
}
renderTable(result.data);
})
.catch(() => {
renderTable(allVisible);
});
},
},
);
this.plugin._ocrProcess = child;
updateProgress();
};
rebuildBatchBtn.addEventListener("click", () =>
runBatch("rebuild")
);
redoBatchBtn.addEventListener("click", () =>
runBatch("redo")
);
updateBatchLabel();
} // end else (visible non-empty)
};
// ── Phase 1: Show cache immediately ──
if (cache) {
const papers = Object.values(cache.papers) as MaintenanceDisplayRow[];
const papers = Object.values(
cache.papers
) as MaintenanceDisplayRow[];
renderTable(papers);
} else {
statusEl.createEl("p", { text: "正在加载 OCR 维护数据…" });
statusEl.createEl("p", {
text: "正在加载 OCR 维护数据…",
});
}
// ── Phase 2: Background refresh ──
@ -2226,14 +2480,22 @@ export class PaperForgeSettingTab extends PluginSettingTab {
renderTable(result.data);
writeMaintenanceCache(vaultPath, {
manifest: {},
papers: Object.fromEntries(result.data.map((p) => [p.key, p])),
papers: Object.fromEntries(
result.data.map(
(p: MaintenanceDisplayRow) => [p.key, p]
)
),
cached_at: new Date().toISOString(),
});
} else if (!cache) {
renderTable(result.data);
writeMaintenanceCache(vaultPath, {
manifest: {},
papers: Object.fromEntries(result.data.map((p) => [p.key, p])),
papers: Object.fromEntries(
result.data.map(
(p: MaintenanceDisplayRow) => [p.key, p]
)
),
cached_at: new Date().toISOString(),
});
}
@ -2242,55 +2504,12 @@ export class PaperForgeSettingTab extends PluginSettingTab {
if (!cache) {
statusEl.empty();
statusEl.createEl("p", {
text: "无法加载 OCR 数据。请确保已安装 paperforge 并运行过 OCR。",
text:
"无法加载 OCR 数据。请确保已安装 paperforge 并运行过 OCR。",
cls: "setting-item-description",
});
}
});
// ── Global operations ──
containerEl.createEl("hr");
containerEl.createEl("h3", { text: "全局操作" });
const globalActions = containerEl.createEl("div", {
cls: "pf-maint-global",
});
const rebuildIndexBtn = globalActions.createEl("button", {
text: "重建搜索索引",
});
rebuildIndexBtn.addEventListener("click", () => {
new Notice("正在重建搜索索引…");
execFile(
py.path,
[
...(py.extraArgs || []),
"-m",
"paperforge",
"embed",
"build",
"--force",
],
{ cwd: vaultPath, timeout: 300000, windowsHide: true },
() => {
new Notice("搜索索引重建完成。");
}
);
});
const rebuildMemBtn = globalActions.createEl("button", {
text: "重建记忆库",
});
rebuildMemBtn.addEventListener("click", () => {
new Notice("正在重建记忆库…");
execFile(
py.path,
[...(py.extraArgs || []), "-m", "paperforge", "repair", "--fix"],
{ cwd: vaultPath, timeout: 120000, windowsHide: true },
() => {
new Notice("记忆库重建完成。");
}
);
});
}
_renderReleaseNotesTab(containerEl: HTMLElement) {

160
paperforge/plugin/styles.css
View file

@ -2684,6 +2684,166 @@
border-color: var(--background-modifier-border-hover);
}
/* ── Maintenance filter tabs ── */
.pf-maint-filters {
display: flex;
gap: 4px;
margin-bottom: 12px;
padding: 2px;
background: var(--background-secondary);
border-radius: 6px;
width: fit-content;
}
.pf-maint-filter {
padding: 4px 14px;
font-size: 12px;
font-weight: 500;
color: var(--text-muted);
background: transparent;
border: none;
border-radius: 4px;
cursor: pointer;
transition: background 0.15s, color 0.15s;
}
.pf-maint-filter:hover {
color: var(--text-normal);
background: var(--background-modifier-hover);
}
.pf-maint-filter.active {
color: var(--text-normal);
background: var(--background-primary);
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.06);
}
/* ── OCR batch progress bar ── */
.pf-maint-progress {
display: flex;
align-items: center;
gap: 10px;
padding: 10px 12px;
margin-bottom: 12px;
background: var(--background-primary);
border: 1px solid var(--background-modifier-border);
border-radius: 8px;
}
.pf-maint-progress .paperforge-progress-track {
flex: 1;
}
.pf-maint-progress-text {
font-size: 11px;
color: var(--text-muted);
white-space: nowrap;
}
.pf-maint-progress-key {
font-size: 10px;
color: var(--text-faint);
font-family: var(--font-monospace);
}
/* ── Batch action bar ── */
.pf-maint-batch-bar {
display: flex;
align-items: center;
gap: 8px;
margin-top: 10px;
padding: 8px 0;
}
.pf-maint-batch-bar button {
padding: 5px 14px;
font-size: 12px;
font-weight: 500;
border-radius: 6px;
cursor: pointer;
transition: opacity 0.15s;
}
.pf-maint-batch-bar button.mod-cta {
background: var(--interactive-accent);
color: var(--text-on-accent);
border: none;
}
.pf-maint-batch-bar button.mod-cta:hover:not(:disabled) {
opacity: 0.85;
}
.pf-maint-batch-bar button:disabled {
opacity: 0.4;
cursor: default;
}
.pf-maint-batch-bar .mod-warning {
background: var(--background-modifier-error);
color: var(--text-on-accent);
border: none;
}
.pf-maint-batch-label {
font-size: 11px;
color: var(--text-muted);
margin-left: auto;
}
/* ── Per-row action buttons ── */
.pf-maint-actions {
display: flex;
gap: 4px;
white-space: nowrap;
}
.pf-maint-action-btn {
padding: 2px 8px;
font-size: 11px;
font-weight: 500;
border: 1px solid var(--background-modifier-border);
border-radius: 4px;
background: var(--background-secondary);
color: var(--text-normal);
cursor: pointer;
transition: background 0.15s;
}
.pf-maint-action-btn:hover:not(:disabled) {
background: var(--background-modifier-hover);
}
.pf-maint-action-btn:disabled {
opacity: 0.4;
cursor: default;
}
.pf-maint-action-btn.rebuild {
color: var(--interactive-accent);
}
.pf-maint-action-btn.redo {
color: var(--text-warning);
}
/* ── Paper info cell ── */
.pf-maint-paper-info {
display: flex;
flex-direction: column;
gap: 1px;
}
.pf-maint-paper-title {
font-size: 12px;
font-weight: 500;
color: var(--text-normal);
line-height: 1.3;
max-width: 240px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.pf-maint-paper-key {
font-size: 10px;
color: var(--text-faint);
font-family: var(--font-monospace);
line-height: 1.3;
}
/* ── Status reason cell ── */
.pf-maint-reason {
font-size: 11px;
color: var(--text-muted);
line-height: 1.35;
max-width: 180px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
/* ==========================================================================
SECTION 41 — Global Settings Tab Polish (Vercel-inspired)
========================================================================== */

289
paperforge/plugin/tests/progress-parser.test.ts Normal file
View file

@ -0,0 +1,289 @@
/**
* Vitest tests for progress-parser.ts — chunk boundary safety,
* prefix matching for EMBED / OCR_REBUILD / OCR_REDO.
*/
import { describe, expect, it } from "vitest";
import {
processProgressChunk,
parseProgressLine,
} from "../src/services/progress-parser";
// ── processProgressChunk ──
describe("processProgressChunk", () => {
it("parses OCR_REBUILD_START:total", () => {
const { events, buffer } = processProgressChunk("OCR_REBUILD_START:5\n", "");
expect(events).toHaveLength(1);
expect(events[0]).toEqual({
prefix: "OCR_REBUILD",
event: "START",
total: 5,
});
expect(buffer).toBe("");
});
it("parses OCR_REBUILD_PROGRESS:current:total:key", () => {
const { events, buffer } = processProgressChunk(
"OCR_REBUILD_PROGRESS:1:5:some-key\n",
"",
);
expect(events).toHaveLength(1);
expect(events[0]).toEqual({
prefix: "OCR_REBUILD",
event: "PROGRESS",
current: 1,
total: 5,
key: "some-key",
});
expect(buffer).toBe("");
});
it("parses OCR_REBUILD_DONE exact match", () => {
const { events } = processProgressChunk("OCR_REBUILD_DONE\n", "");
expect(events).toHaveLength(1);
expect(events[0]).toEqual({
prefix: "OCR_REBUILD",
event: "DONE",
});
});
it("parses multiple OCR_REDO tokens from one chunk", () => {
const chunk =
"OCR_REDO_START:3\nOCR_REDO_PROGRESS:1:3:key-1\nOCR_REDO_DONE\n";
const { events, buffer } = processProgressChunk(chunk, "");
expect(events).toHaveLength(3);
expect(events[0]).toEqual({
prefix: "OCR_REDO",
event: "START",
total: 3,
});
expect(events[1]).toEqual({
prefix: "OCR_REDO",
event: "PROGRESS",
current: 1,
total: 3,
key: "key-1",
});
expect(events[2]).toEqual({ prefix: "OCR_REDO", event: "DONE" });
expect(buffer).toBe("");
});
it("parses EMBED tokens for backward compatibility", () => {
const chunk =
"EMBED_START:10\nEMBED_PROGRESS:2:10:paper-abc\nEMBED_DONE\n";
const { events, buffer } = processProgressChunk(chunk, "");
expect(events).toHaveLength(3);
expect(events[0]).toEqual({ prefix: "EMBED", event: "START", total: 10 });
expect(events[1]).toEqual({
prefix: "EMBED",
event: "PROGRESS",
current: 2,
total: 10,
key: "paper-abc",
});
expect(events[2]).toEqual({ prefix: "EMBED", event: "DONE" });
expect(buffer).toBe("");
});
it("buffers an incomplete last line", () => {
const { events, buffer } = processProgressChunk(
"OCR_REBUILD_PROGRESS:1:5:",
"",
);
expect(events).toHaveLength(0);
expect(buffer).toBe("OCR_REBUILD_PROGRESS:1:5:");
});
it("resumes from buffer on the next chunk", () => {
const first = processProgressChunk("OCR_REBUILD_PROGRESS:1:", "");
expect(first.events).toHaveLength(0);
expect(first.buffer).toBe("OCR_REBUILD_PROGRESS:1:");
const second = processProgressChunk("5:paper-key\n", first.buffer);
expect(second.events).toHaveLength(1);
expect(second.events[0]).toEqual({
prefix: "OCR_REBUILD",
event: "PROGRESS",
current: 1,
total: 5,
key: "paper-key",
});
expect(second.buffer).toBe("");
});
it("passes through non-token lines silently and keeps buffer", () => {
const { events, buffer } = processProgressChunk(
"normal stdout line\nanother\n",
"",
);
expect(events).toHaveLength(0);
expect(buffer).toBe("");
});
it("handles interleaved normal output and tokens", () => {
const chunk =
"info: starting\nOCR_REBUILD_START:2\nsome log\nOCR_REBUILD_PROGRESS:1:2:k1\nmore output\nOCR_REBUILD_DONE\ndone\n";
const { events, buffer } = processProgressChunk(chunk, "");
expect(events).toHaveLength(3);
expect(events[0]).toEqual({ prefix: "OCR_REBUILD", event: "START", total: 2 });
expect(events[1]).toEqual({
prefix: "OCR_REBUILD",
event: "PROGRESS",
current: 1,
total: 2,
key: "k1",
});
expect(events[2]).toEqual({ prefix: "OCR_REBUILD", event: "DONE" });
expect(buffer).toBe("");
});
it("handles empty chunk gracefully", () => {
const { events, buffer } = processProgressChunk("", "existing-buffer");
expect(events).toHaveLength(0);
expect(buffer).toBe("existing-buffer");
});
it("uses empty buffer on first call when omitted", () => {
// With empty string buffer
const { events, buffer } = processProgressChunk("EMBED_DONE\n", "");
expect(events).toHaveLength(1);
expect(buffer).toBe("");
});
it("parses EMBED_START split at prefix boundary", () => {
// First chunk has partial prefix
const first = processProgressChunk("EM", "");
expect(first.events).toHaveLength(0);
expect(first.buffer).toBe("EM");
// Second chunk completes the token
const second = processProgressChunk("BED_START:3\n", first.buffer);
expect(second.events).toHaveLength(1);
expect(second.events[0]).toEqual({
prefix: "EMBED",
event: "START",
total: 3,
});
expect(second.buffer).toBe("");
});
it("parses OCR_REBUILD_START split across chunks", () => {
const first = processProgressChunk("OCR_REB", "");
expect(first.events).toHaveLength(0);
expect(first.buffer).toBe("OCR_REB");
const second = processProgressChunk("UILD_START:5\n", first.buffer);
expect(second.events).toHaveLength(1);
expect(second.events[0]).toEqual({
prefix: "OCR_REBUILD",
event: "START",
total: 5,
});
expect(second.buffer).toBe("");
});
it("parses EMBED_PROGRESS split between prefix and colon", () => {
const first = processProgressChunk("EMBED_PROGRESS", "");
expect(first.events).toHaveLength(0);
expect(first.buffer).toBe("EMBED_PROGRESS");
const second = processProgressChunk(":2:5:my-key\n", first.buffer);
expect(second.events).toHaveLength(1);
expect(second.events[0]).toEqual({
prefix: "EMBED",
event: "PROGRESS",
current: 2,
total: 5,
key: "my-key",
});
expect(second.buffer).toBe("");
});
it("parses EMBED_DONE across two chunks", () => {
const first = processProgressChunk("EMBED_D", "");
expect(first.events).toHaveLength(0);
expect(first.buffer).toBe("EMBED_D");
const second = processProgressChunk("ONE\n", first.buffer);
expect(second.events).toHaveLength(1);
expect(second.events[0]).toEqual({
prefix: "EMBED",
event: "DONE",
});
expect(second.buffer).toBe("");
});
it("parses EMBED tokens with interleaved normal output across chunks", () => {
// First chunk: START is complete, PROGRESS line is split without trailing \n
const first = processProgressChunk(
"log line\nEMBED_START:2\nmore log\nEMBED_PROGRESS:1:2:",
"",
);
expect(first.events).toHaveLength(1);
expect(first.events[0]).toEqual({ prefix: "EMBED", event: "START", total: 2 });
// Buffer holds only the last incomplete fragment
expect(first.buffer).toBe("EMBED_PROGRESS:1:2:");
// Second chunk completes the PROGRESS line and has complete DONE
const second = processProgressChunk("k1\nEMBED_DONE\n", first.buffer);
expect(second.events).toHaveLength(2);
expect(second.events[0]).toEqual({
prefix: "EMBED", event: "PROGRESS",
current: 1, total: 2, key: "k1",
});
expect(second.events[1]).toEqual({ prefix: "EMBED", event: "DONE" });
expect(second.buffer).toBe("");
});
it("parses all EMBED tokens across chunk boundaries", () => {
const first = processProgressChunk("EMBED_START:2\nEMBED_PR", "");
expect(first.events).toHaveLength(1);
expect(first.events[0]).toEqual({ prefix: "EMBED", event: "START", total: 2 });
expect(first.buffer).toBe("EMBED_PR");
const second = processProgressChunk("OGRESS:1:2:k1\nEMBED_DONE\n", first.buffer);
expect(second.events).toHaveLength(2);
expect(second.events[0]).toEqual({
prefix: "EMBED", event: "PROGRESS",
current: 1, total: 2, key: "k1",
});
expect(second.events[1]).toEqual({ prefix: "EMBED", event: "DONE" });
expect(second.buffer).toBe("");
});
});
// ── parseProgressLine ──
describe("parseProgressLine", () => {
it("returns START for OCR_REBUILD_START:5", () => {
expect(parseProgressLine("OCR_REBUILD_START:5")).toEqual({
prefix: "OCR_REBUILD",
event: "START",
total: 5,
});
});
it("returns PROGRESS for full token", () => {
expect(parseProgressLine("OCR_REDO_PROGRESS:3:10:paper-z")).toEqual({
prefix: "OCR_REDO",
event: "PROGRESS",
current: 3,
total: 10,
key: "paper-z",
});
});
it("returns DONE for exact match", () => {
expect(parseProgressLine("EMBED_DONE")).toEqual({
prefix: "EMBED",
event: "DONE",
});
});
it("returns null for non-token line", () => {
expect(parseProgressLine("hello world")).toBeNull();
});
it("returns null for empty string", () => {
expect(parseProgressLine("")).toBeNull();
});
});

145
paperforge/worker/ocr.py
View file

@ -2102,6 +2102,150 @@ def _rewrite_note_fields(text: str, **fields: object) -> str:
return updated
def _find_notes_by_key(lit_root: Path | None) -> dict[str, Path]:
"""Scan literature directory and build {zotero_key: note_path} map."""
notes: dict[str, Path] = {}
if not lit_root or not lit_root.exists():
return notes
for note_file in sorted(lit_root.rglob("*.md")):
if note_file.name in ("fulltext.md", "deep-reading.md", "discussion.md"):
continue
try:
text = note_file.read_text(encoding="utf-8")
except Exception:
continue
key_match = re.search(r"^zotero_key:\s*(.+)$", text, re.MULTILINE)
if not key_match:
continue
zkey = key_match.group(1).strip().strip('"').strip("'")
if not zkey or not re.match(r"^[A-Za-z0-9]{8}$", zkey):
continue
existing = notes.get(zkey)
if existing is None or note_file.name == f"{zkey}.md":
notes[zkey] = note_file
return notes
def redo_papers_for_keys(
vault: Path,
keys: list[str],
*,
verbose: bool = False,
progress_callback: Callable[[str], None] | None = None,
stop_check: Callable[[], bool] | None = None,
) -> dict:
"""Redo specific papers: full artifact delete + OCR run + post-check.
Processes papers sequentially so per-paper progress and cooperative
stop are natural. Each paper goes through:
1. Delete derived artifacts and workspace fulltexts
2. Rewrite note as pending
3. Run OCR for this single paper
4. Check result, update note, refresh index
Args:
vault: Vault root.
keys: Zotero citation keys to redo.
verbose: Log additional details.
progress_callback: Called with key after each paper's cycle completes.
stop_check: Called before each paper; return True to stop.
Returns:
dict with success_keys, failed_keys, exit_code.
"""
from paperforge.worker._utils import pipeline_paths
paths = pipeline_paths(vault)
ocr_root = paths.get("ocr")
lit_root = paths.get("literature")
notes_by_key = _find_notes_by_key(lit_root)
success_keys: list[str] = []
failed_keys: list[str] = []
_ocr_exit_code = 0
for key in keys:
if stop_check is not None and stop_check():
break
note_file = notes_by_key.get(key)
if note_file is None:
logger.warning("Redo: note not found for %s, counting as failed", key)
failed_keys.append(key)
_ocr_exit_code = 1
if progress_callback is not None:
progress_callback(key)
continue
try:
original_text = note_file.read_text(encoding="utf-8")
except Exception:
logger.warning("Redo: cannot read note for %s, counting as failed", key)
failed_keys.append(key)
if progress_callback is not None:
progress_callback(key)
_ocr_exit_code = 1
continue
# ── Guard: nopdf papers must not be touched ──
meta_path = ocr_root / key / "meta.json" if ocr_root else None
_existing_meta = read_json(meta_path) if meta_path and meta_path.exists() else {}
if str(_existing_meta.get("ocr_status", "") or "").strip().lower() == "nopdf":
logger.warning("Redo: %s is nopdf — cannot redo (missing PDF), skipping", key)
failed_keys.append(key)
_ocr_exit_code = 1
if progress_callback is not None:
progress_callback(key)
continue
# ── Phase 1: delete artifacts ──
for wf in _workspace_fulltext_candidates(lit_root, key):
wf.unlink(missing_ok=True)
ocr_dir = ocr_root / key if ocr_root else None
if ocr_dir and ocr_dir.exists():
shutil.rmtree(ocr_dir)
# ── Phase 2: rewrite note as pending ──
text = _rewrite_note_fields(
original_text,
do_ocr=True, ocr_status="pending", fulltext_md_path="", ocr_redo=True,
)
note_file.write_text(text, encoding="utf-8")
# ── Phase 3: run OCR for this paper ──
_paper_exit = run_ocr(vault, verbose=verbose, no_progress=True, selected_keys={key})
if _paper_exit != 0:
_ocr_exit_code = _paper_exit
# ── Phase 4: post-OCR check ──
meta_path = ocr_root / key / "meta.json" if ocr_root else None
meta = read_json(meta_path) if meta_path and meta_path.exists() else {}
status, _error = validate_ocr_meta(paths, meta) if meta else ("pending", "")
current_text = note_file.read_text(encoding="utf-8")
current_text = _rewrite_note_fields(current_text, ocr_status=status)
if status == "done":
current_text = _rewrite_note_fields(current_text, ocr_redo=False)
success_keys.append(key)
else:
current_text = _rewrite_note_fields(current_text, ocr_redo=True)
failed_keys.append(key)
_ocr_exit_code = _ocr_exit_code or 1
note_file.write_text(current_text, encoding="utf-8")
refresh_index_entry(vault, key)
if progress_callback is not None:
progress_callback(key)
return {
"success_keys": success_keys,
"failed_keys": failed_keys,
"exit_code": _ocr_exit_code,
}
def ocr_redo_papers(vault: Path, dry_run: bool = False, verbose: bool = False, no_progress: bool = False) -> int:
"""Scan for papers with ocr_redo: true, reset and immediately rerun OCR.
@ -2207,6 +2351,7 @@ def ocr_redo_papers(vault: Path, dry_run: bool = False, verbose: bool = False, n
return exit_code
def run_ocr(
vault: Path, verbose: bool = False, no_progress: bool = False, selected_keys: set[str] | None = None
) -> int:

23
paperforge/worker/ocr_rebuild.py
View file

@ -4,6 +4,7 @@ import datetime
import json
import logging
from pathlib import Path
from collections.abc import Callable
from paperforge.core.io import read_json, write_json
@ -587,7 +588,10 @@ def _rebuild_one_paper(vault: Path, key: str) -> dict:
_phase5_finalize(resolved, structured, rendered, span_meta_patch, health_overall=health_overall)
return {"key": key, "status": "ok"}
def _run_parallel_rebuild(vault: Path, keys: list[str], workers: int) -> list[dict]:
def _run_parallel_rebuild(
vault: Path, keys: list[str], workers: int,
on_progress: Callable[[str], None] | None = None,
) -> list[dict]:
"""Run rebuild in parallel using a process pool."""
from concurrent.futures import ProcessPoolExecutor, as_completed
@ -601,15 +605,20 @@ def _run_parallel_rebuild(vault: Path, keys: list[str], workers: int) -> list[di
results.append(result)
except Exception as e:
results.append({"key": key, "status": "failed", "error": str(e)})
if on_progress is not None:
on_progress(key)
return results
def run_derived_rebuild_for_keys(
vault: Path,
keys: list[str],
progress_bar=None,
checkpoint_dir: Path | None = None,
parallel: int = 4,
on_progress: Callable[[str], None] | None = None,
stop_check: Callable[[], bool] | None = None,
) -> dict:
"""Run derived-layer rebuild for the given paper keys without raw OCR rerun.
@ -625,6 +634,9 @@ def run_derived_rebuild_for_keys(
progress_bar: Optional progress bar wrapper (tqdm-style).
checkpoint_dir: Deprecated, kept for backward compatibility.
parallel: Number of parallel workers (0 = serial). Default 4.
on_progress: Optional callback called with key after each paper completes.
stop_check: Optional callable returning True if stop was requested.
Checked before starting the next paper (serial path only).
"""
if not keys:
return {"rebuild_count": 0}
@ -632,18 +644,25 @@ def run_derived_rebuild_for_keys(
workers = int(parallel) if parallel else 0
if workers > 0 and len(keys) > 1:
results = _run_parallel_rebuild(vault, keys, workers)
# Parallel path: does not support stop_check — all futures submitted at once.
results = _run_parallel_rebuild(vault, keys, workers, on_progress=on_progress)
return {"rebuild_count": sum(1 for r in results if r.get("status") == "ok")}
rebuilt_count = 0
keys_iter = progress_bar(keys, desc="OCR rebuild") if progress_bar else keys
for key in keys_iter:
if stop_check is not None and stop_check():
break
result = _rebuild_one_paper(vault, key)
if result.get("status") == "ok":
rebuilt_count += 1
if on_progress is not None:
on_progress(key)
return {"rebuild_count": rebuilt_count}
def _enrich_meta_from_paper_note(vault: Path, key: str, meta_path: Path) -> None:
"""Read the paper note's Zotero frontmatter and inject into meta.json.

500
tests/cli/test_ocr_progress_contracts.py Normal file
View file

@ -0,0 +1,500 @@
"""CLI contract tests for OCR rebuild/redo progress tokens.
Tests the streaming progress token contract defined in issue #64:
- OCR_REBUILD_START:{total}
- OCR_REBUILD_PROGRESS:{current}:{total}:{key}
- OCR_REBUILD_DONE
- OCR_REDO_START:{total}
- OCR_REDO_PROGRESS:{current}:{total}:{key}
- OCR_REDO_DONE
Batch mode only (multiple keys). Single-key calls remain silent.
Dry-run must not emit progress tokens.
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
# ── Helpers ──────────────────────────────────────────────────────────────
def _make_minimal_vault(root: Path) -> Path:
"""Create a minimal vault with paperforge.json for path resolution."""
vault = root / "vault"
vault.mkdir(parents=True)
cfg = {
"system_dir": "System",
"resources_dir": "Resources",
"literature_dir": "Literature",
"control_dir": "LiteratureControl",
"base_dir": "Bases",
"skill_dir": ".opencode/skills",
"zotero_dir": "System/Zotero",
}
(vault / "paperforge.json").write_text(json.dumps(cfg), encoding="utf-8")
dirs = [
"System/PaperForge/ocr",
"System/PaperForge/exports",
"System/PaperForge/indexes",
"System/PaperForge/config",
"System/Zotero/storage",
"Resources/Literature",
"Resources/LiteratureControl",
"Bases",
".opencode/skills/literature-qa/scripts",
]
for d in dirs:
(vault / d).mkdir(parents=True)
return vault
def _add_ocr_meta(vault: Path, key: str, status: str = "pending") -> None:
"""Create OCR meta.json for a given paper key."""
meta_dir = vault / "System/PaperForge/ocr" / key
meta_dir.mkdir(parents=True, exist_ok=True)
meta = {
"zotero_key": key,
"ocr_status": status,
"page_count": 1,
"generated_at": "2026-07-01T00:00:00+00:00",
}
(meta_dir / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
def _add_literature_note(vault: Path, key: str, title: str = "Test Paper") -> None:
"""Create a minimal literature note with zotero_key in frontmatter."""
lit_dir = vault / "Resources" / "Literature"
lit_dir.mkdir(parents=True, exist_ok=True)
note = lit_dir / f"{key}.md"
note.write_text(
f"---\nzotero_key: \"{key}\"\ntitle: \"{title}\"\n---\n\nPaper content.\n",
encoding="utf-8",
)
def _mock_run_ocr(monkeypatch) -> None:
"""Mock run_ocr to create minimal meta.json with done status (no API calls)."""
import json
def _fake_run_ocr(vault: Path, **kwargs: object) -> int:
keys: set[str] = kwargs.get("selected_keys") or set()
from paperforge.worker._utils import pipeline_paths
paths = pipeline_paths(vault)
ocr_root = paths.get("ocr")
if ocr_root:
for key in keys:
meta_dir = ocr_root / key
meta_dir.mkdir(parents=True, exist_ok=True)
(meta_dir / "meta.json").write_text(
json.dumps({"zotero_key": key, "ocr_status": "done"}),
encoding="utf-8",
)
return 0
monkeypatch.setattr("paperforge.worker.ocr.run_ocr", _fake_run_ocr)
def _mock_validate_ocr_meta(monkeypatch) -> None:
"""Mock validate_ocr_meta to return done status."""
monkeypatch.setattr(
"paperforge.worker.ocr.validate_ocr_meta",
lambda _paths, _meta: ("done", ""),
)
# ── Redo Progress Token Tests ────────────────────────────────────────────
class TestOcrRedoProgressTokens:
"""OCR_REDO_* progress token contract tests."""
_K1 = "KEY00001"
_K2 = "KEY00002"
_K3 = "KEY00003"
def test_redo_multi_key_emits_start_progress_done(
self, capsys, tmp_path, monkeypatch
):
"""Multiple redo keys emit OCR_REDO_START, PROGRESS per key, DONE."""
_mock_run_ocr(monkeypatch)
_mock_validate_ocr_meta(monkeypatch)
vault = _make_minimal_vault(tmp_path)
for k in (self._K1, self._K2, self._K3):
_add_literature_note(vault, k)
from paperforge.commands.ocr import _run_ocr_redo
rc = _run_ocr_redo(vault, keys=[self._K1, self._K2, self._K3])
assert rc == 0
captured = capsys.readouterr().out
lines = [l for l in captured.split("\n") if l.strip()]
# Token order: START, then PROGRESS per key, summary, DONE
assert lines[0] == "OCR_REDO_START:3"
progress_lines = [l for l in lines if l.startswith("OCR_REDO_PROGRESS")]
assert len(progress_lines) == 3
assert progress_lines[0] == "OCR_REDO_PROGRESS:1:3:KEY00001"
assert progress_lines[1] == "OCR_REDO_PROGRESS:2:3:KEY00002"
assert progress_lines[2] == "OCR_REDO_PROGRESS:3:3:KEY00003"
assert "Redo OCR done" in captured
assert "OCR_REDO_DONE" in captured
def test_redo_single_key_no_tokens(self, capsys, tmp_path, monkeypatch):
"""Single redo key emits no progress tokens."""
_mock_run_ocr(monkeypatch)
_mock_validate_ocr_meta(monkeypatch)
vault = _make_minimal_vault(tmp_path)
_add_literature_note(vault, self._K1)
from paperforge.commands.ocr import _run_ocr_redo
rc = _run_ocr_redo(vault, keys=[self._K1])
assert rc == 0
captured = capsys.readouterr().out
assert "OCR_REDO_START" not in captured
assert "OCR_REDO_PROGRESS" not in captured
assert "OCR_REDO_DONE" not in captured
assert "Redo OCR done=1" in captured
def test_redo_dry_run_no_tokens(self, capsys, tmp_path, monkeypatch):
"""Dry-run redo emits no progress tokens."""
_mock_run_ocr(monkeypatch)
_mock_validate_ocr_meta(monkeypatch)
vault = _make_minimal_vault(tmp_path)
_add_literature_note(vault, self._K1)
_add_literature_note(vault, self._K2)
from paperforge.commands.ocr import _run_ocr_redo
rc = _run_ocr_redo(vault, keys=[self._K1, self._K2], dry_run=True)
assert rc == 0
captured = capsys.readouterr().out
# Multi-key dry-run is not batch (batch = total > 1 and not dry_run)
assert "OCR_REDO_START" not in captured
assert "OCR_REDO_PROGRESS" not in captured
assert "OCR_REDO_DONE" not in captured
assert "Would redo" in captured
def test_redo_prefix_separate_from_rebuild(self, capsys, tmp_path, monkeypatch):
"""OCR_REDO prefix is distinct from OCR_REBUILD."""
_mock_run_ocr(monkeypatch)
_mock_validate_ocr_meta(monkeypatch)
vault = _make_minimal_vault(tmp_path)
for k in (self._K1, self._K2, self._K3):
_add_literature_note(vault, k)
from paperforge.commands.ocr import _run_ocr_redo
_run_ocr_redo(vault, keys=[self._K1, self._K2, self._K3])
captured = capsys.readouterr().out
assert "OCR_REDO_START" in captured
assert "OCR_REDO_PROGRESS" in captured
assert "OCR_REDO_DONE" in captured
assert "OCR_REBUILD_START" not in captured
# ── Rebuild Progress Token Tests ─────────────────────────────────────────
class TestOcrRebuildProgressTokens:
"""OCR_REBUILD_* progress token contract tests."""
def _setup_mock_selection_and_worker(
self, vault: Path, keys: list[str], monkeypatch
) -> None:
"""Mock selection and worker for rebuild, calling on_progress."""
from paperforge.worker.ocr_maintenance import OCRMaintenanceRow as Row
rows = [Row(
key=k,
title=f"Paper {k}",
title_full=f"Full Title for Paper {k}",
status="done",
health="good",
pages=5,
blocks=100,
version="2.0",
finished_at="2026-07-01",
rebuild_finished_at="",
figures=0,
tables=0,
model="test",
structured_content_hash=f"hash_{k}",
can_rebuild=True,
) for k in keys]
monkeypatch.setattr(
"paperforge.worker.ocr_maintenance.collect_maintenance_rows",
lambda _v: rows,
)
def _mock_rebuild(
_vault: Path,
_keys: list[str],
**kwargs: object,
) -> dict:
on_progress = kwargs.get("on_progress")
if on_progress:
for k in _keys:
on_progress(k)
return {"rebuild_count": len(_keys)}
monkeypatch.setattr(
"paperforge.worker.ocr_rebuild.run_derived_rebuild_for_keys",
_mock_rebuild,
)
def test_rebuild_multi_key_emits_start_progress_done(
self, capsys, monkeypatch, tmp_path
):
"""Multiple rebuild keys emit OCR_REBUILD_START, PROGRESS per key, DONE."""
vault = _make_minimal_vault(tmp_path)
keys = ["KEY001", "KEY002", "KEY003"]
self._setup_mock_selection_and_worker(vault, keys, monkeypatch)
from paperforge.commands.ocr import _run_ocr_rebuild
rc = _run_ocr_rebuild(vault, keys=keys)
assert rc == 0
captured = capsys.readouterr().out
lines = [l for l in captured.split("\n") if l.strip()]
assert lines[0] == "OCR_REBUILD_START:3", f"First token: {lines}"
progress_lines = [l for l in lines if "OCR_REBUILD_PROGRESS" in l]
assert len(progress_lines) == 3
assert progress_lines[0] == "OCR_REBUILD_PROGRESS:1:3:KEY001"
assert progress_lines[1] == "OCR_REBUILD_PROGRESS:2:3:KEY002"
assert progress_lines[2] == "OCR_REBUILD_PROGRESS:3:3:KEY003"
assert lines[-1] == "OCR_REBUILD_DONE", f"Last token: {lines}"
def test_rebuild_single_key_no_tokens(self, capsys, monkeypatch, tmp_path):
"""Single rebuild key emits no progress tokens."""
vault = _make_minimal_vault(tmp_path)
key = "KEY001"
self._setup_mock_selection_and_worker(vault, [key], monkeypatch)
from paperforge.commands.ocr import _run_ocr_rebuild
rc = _run_ocr_rebuild(vault, keys=[key])
assert rc == 0
captured = capsys.readouterr().out
assert "OCR_REBUILD_START" not in captured
assert "OCR_REBUILD_PROGRESS" not in captured
assert "OCR_REBUILD_DONE" not in captured
def test_rebuild_dry_run_no_tokens(self, capsys, monkeypatch, tmp_path):
"""Dry-run rebuild emits no progress tokens."""
vault = _make_minimal_vault(tmp_path)
self._setup_mock_selection_and_worker(vault, ["KEY001", "KEY002"], monkeypatch)
from paperforge.commands.ocr import _run_ocr_rebuild
rc = _run_ocr_rebuild(vault, keys=["KEY001", "KEY002"], dry_run=True)
assert rc == 0
captured = capsys.readouterr().out
assert "OCR_REBUILD_START" not in captured
assert "OCR_REBUILD_PROGRESS" not in captured
assert "OCR_REBUILD_DONE" not in captured
assert "Would rebuild" in captured
def test_rebuild_prefix_uses_ocr_rebuild_not_redo(
self, capsys, monkeypatch, tmp_path
):
"""OCR rebuild uses OCR_REBUILD prefix, not OCR_REDO."""
vault = _make_minimal_vault(tmp_path)
keys = ["KEY001", "KEY002"]
self._setup_mock_selection_and_worker(vault, keys, monkeypatch)
from paperforge.commands.ocr import _run_ocr_rebuild
_run_ocr_rebuild(vault, keys=keys)
captured = capsys.readouterr().out
assert "OCR_REBUILD_START" in captured
assert "OCR_REBUILD_PROGRESS" in captured
assert "OCR_REBUILD_DONE" in captured
assert "OCR_REDO_START" not in captured
class TestCooperativeStop:
"""Cooperative stop tests for batch OCR operations."""
def test_rebuild_stop_emits_done_and_exit_code_130(self, capsys, monkeypatch, tmp_path):
"""Rebuild batch stop sets exit 130 and emits DONE with partial progress."""
vault = _make_minimal_vault(tmp_path)
keys = ["KEY001", "KEY002", "KEY003"]
# Set up maintenance rows via mock
from paperforge.worker.ocr_maintenance import OCRMaintenanceRow as Row
rows = [Row(
key=k, title=f"Paper {k}", title_full=f"Full {k}",
status="done", health="good", pages=5, blocks=100,
version="2.0", finished_at="2026-07-01", rebuild_finished_at="",
figures=0, tables=0, model="test",
structured_content_hash=f"hash_{k}", can_rebuild=True,
) for k in keys]
monkeypatch.setattr(
"paperforge.worker.ocr_maintenance.collect_maintenance_rows",
lambda _v: rows,
)
# Mock _rebuild_one_paper so the real run_derived_rebuild_for_keys can run
call_count = [0]
def _mock_rebuild_one(vault, key):
call_count[0] += 1
return {"key": key, "status": "ok"}
monkeypatch.setattr(
"paperforge.worker.ocr_rebuild._rebuild_one_paper",
_mock_rebuild_one,
)
# Mock stop_check to stop after first paper
stop_count = [0]
def _stop_check():
stop_count[0] += 1
return stop_count[0] > 1 # stop before the 2nd call
from paperforge.worker.ocr_rebuild import run_derived_rebuild_for_keys
from paperforge.worker._progress import progress_bar
# Simulate what _run_ocr_rebuild does for batch
print("OCR_REBUILD_START:3")
count = [0]
def _on_progress(key):
count[0] += 1
print(f"OCR_REBUILD_PROGRESS:{count[0]}:3:{key}")
result = run_derived_rebuild_for_keys(
vault, keys,
progress_bar=progress_bar,
parallel=0,
on_progress=_on_progress,
stop_check=_stop_check,
)
print(f"Done. Rebuilt {result['rebuild_count']} paper(s).")
print("OCR_REBUILD_DONE")
captured = capsys.readouterr().out
lines = [l for l in captured.split("\n") if l.strip()]
# Only first key should have been processed
assert result["rebuild_count"] == 1
assert call_count[0] == 1
# Should still emit DONE
assert "OCR_REBUILD_START:3" in captured
assert "OCR_REBUILD_PROGRESS:1:3:KEY001" in captured
assert "OCR_REBUILD_PROGRESS:2:3:KEY002" not in captured
assert "OCR_REBUILD_PROGRESS:3:3:KEY003" not in captured
assert "OCR_REBUILD_DONE" in captured
def test_redo_stop_emits_done_and_partial_progress(self, capsys, monkeypatch, tmp_path):
"""Redo batch stop emits DONE with partial progress and exit 130."""
_mock_run_ocr(monkeypatch)
_mock_validate_ocr_meta(monkeypatch)
vault = _make_minimal_vault(tmp_path)
_add_literature_note(vault, "KEY00001")
_add_literature_note(vault, "KEY00002")
_add_literature_note(vault, "KEY00003")
from paperforge.commands.ocr import _run_ocr_redo
# Mock _make_cooperative_stop to stop after first paper
class _StopChecker:
def __init__(self):
self._call_count = 0
def __call__(self):
self._call_count += 1
return self._call_count >= 2 # stop before 2nd paper
checker = _StopChecker()
def _fake_make_cooperative_stop():
return (checker, lambda: None)
monkeypatch.setattr(
"paperforge.commands.ocr._make_cooperative_stop",
_fake_make_cooperative_stop,
)
rc = _run_ocr_redo(vault, keys=["KEY00001", "KEY00002", "KEY00003"])
assert rc == 130, f"Expected exit code 130 for stopped batch, got {rc}"
captured = capsys.readouterr().out
lines = [l for l in captured.split("\n") if l.strip()]
assert lines[0] == "OCR_REDO_START:3"
assert "OCR_REDO_PROGRESS:1:3:KEY00001" in captured
assert "OCR_REDO_PROGRESS:2:3:KEY00002" not in captured
assert "OCR_REDO_DONE" in captured
assert "Batch stopped" in captured
# On Windows os.kill(pid, SIGINT) does not trigger Python signal
# handlers the same way; this test is platform-dependent and skipped.
def test_make_cooperative_stop_signal_handler(self):
"""_make_cooperative_stop installs handler and tracks flag."""
import sys
if sys.platform == "win32":
pytest.skip("SIGINT signal handling differs on Windows")
from paperforge.commands.ocr import _make_cooperative_stop
is_stopped, restore = _make_cooperative_stop()
# Initially not stopped
assert not is_stopped()
# Simulate SIGINT
import signal
import os
os.kill(os.getpid(), signal.SIGINT)
# Flag should be set
assert is_stopped()
# Restore original handler
restore()
# After restore, new call installs fresh handler
is_stopped2, restore2 = _make_cooperative_stop()
assert not is_stopped2()
restore2()
def test_make_cooperative_stop_stdin(self, monkeypatch):
"""_make_cooperative_stop reads PAPERFORGE_STOP from stdin to set flag."""
import os
import time
r_fd, w_fd = os.pipe()
fake_stdin = os.fdopen(r_fd, "r", encoding="utf-8")
monkeypatch.setattr("sys.stdin", fake_stdin)
from paperforge.commands.ocr import _make_cooperative_stop
is_stopped, restore = _make_cooperative_stop()
# Initially not stopped
assert not is_stopped()
# Write stop command to the pipe
os.write(w_fd, b"PAPERFORGE_STOP\n")
os.close(w_fd)
# Give reader thread time to process
time.sleep(0.2)
# Flag should be set
assert is_stopped(), "Stdin stop command should set flag"
restore()

151
tests/test_ocr_maintenance.py
View file

@ -631,54 +631,79 @@ class TestOcrListKeys:
class TestOcrRedoKeyed:
"""_run_ocr_redo with keys parameter."""
def test_redo_single_key(self, tmp_path) -> None:
def test_redo_single_key(self, tmp_path, monkeypatch) -> None:
"""Single key redo delegates to worker with full cycle."""
from paperforge.commands.ocr import _run_ocr_redo
ocr_root = _ocr_path(tmp_path)
(ocr_root / "KEY1").mkdir(parents=True)
meta = {"zotero_key": "KEY1", "ocr_status": "done", "ocr_job_id": "job-123"}
(ocr_root / "KEY1" / "meta.json").write_text(json.dumps(meta), encoding="utf-8")
result = _run_ocr_redo(tmp_path, keys=["KEY1"])
assert result == 0
reloaded = json.loads(
(ocr_root / "KEY1" / "meta.json").read_text(encoding="utf-8")
from tests.cli.test_ocr_progress_contracts import (
_make_minimal_vault,
_add_literature_note,
_mock_run_ocr,
)
assert reloaded["ocr_status"] == "pending"
assert reloaded["ocr_job_id"] == ""
_mock_run_ocr(monkeypatch)
monkeypatch.setattr("paperforge.worker.ocr.validate_ocr_meta",
lambda _p, _m: ("done", ""))
def test_redo_multiple_keys(self, tmp_path) -> None:
from paperforge.commands.ocr import _run_ocr_redo
vault = _make_minimal_vault(tmp_path)
_add_literature_note(vault, "KEY00001")
ocr_root = _ocr_path(tmp_path)
for key in ["KEY1", "KEY2"]:
(ocr_root / key).mkdir(parents=True)
meta = {"zotero_key": key, "ocr_status": "done", "ocr_job_id": f"job-{key}"}
(ocr_root / key / "meta.json").write_text(json.dumps(meta), encoding="utf-8")
result = _run_ocr_redo(tmp_path, keys=["KEY1", "KEY2"])
result = _run_ocr_redo(vault, keys=["KEY00001"])
assert result == 0
for key in ["KEY1", "KEY2"]:
reloaded = json.loads(
(ocr_root / key / "meta.json").read_text(encoding="utf-8")
)
assert reloaded["ocr_status"] == "pending"
assert reloaded["ocr_job_id"] == ""
# Worker should have created meta.json with done status
meta_path = vault / "System/PaperForge/ocr" / "KEY00001" / "meta.json"
assert meta_path.exists(), "Meta should exist after redo cycle"
import json
meta = json.loads(meta_path.read_text(encoding="utf-8"))
assert meta.get("ocr_status") == "done"
def test_redo_dry_run(self, tmp_path) -> None:
def test_redo_multiple_keys(self, tmp_path, monkeypatch) -> None:
"""Multiple keys redo delegates to worker for each key."""
from paperforge.commands.ocr import _run_ocr_redo
from tests.cli.test_ocr_progress_contracts import (
_make_minimal_vault,
_add_literature_note,
_mock_run_ocr,
)
_mock_run_ocr(monkeypatch)
monkeypatch.setattr("paperforge.worker.ocr.validate_ocr_meta",
lambda _p, _m: ("done", ""))
ocr_root = _ocr_path(tmp_path)
(ocr_root / "KEY1").mkdir(parents=True)
meta = {"zotero_key": "KEY1", "ocr_status": "done", "ocr_job_id": "job-123"}
meta_path = ocr_root / "KEY1" / "meta.json"
meta_path.write_text(json.dumps(meta), encoding="utf-8")
vault = _make_minimal_vault(tmp_path)
for k in ("KEY00001", "KEY00002"):
_add_literature_note(vault, k)
_run_ocr_redo(tmp_path, keys=["KEY1"], dry_run=True)
result = _run_ocr_redo(vault, keys=["KEY00001", "KEY00002"])
assert result == 0
# Both papers should have meta.json
for k in ("KEY00001", "KEY00002"):
meta_path = vault / "System/PaperForge/ocr" / k / "meta.json"
assert meta_path.exists(), f"Meta for {k} should exist"
def test_redo_dry_run(self, tmp_path, monkeypatch) -> None:
"""Dry-run redo prints without modifying artifacts."""
from paperforge.commands.ocr import _run_ocr_redo
from tests.cli.test_ocr_progress_contracts import (
_make_minimal_vault,
_add_literature_note,
)
reloaded = json.loads(meta_path.read_text(encoding="utf-8"))
assert reloaded["ocr_status"] == "done"
assert reloaded["ocr_job_id"] == "job-123"
vault = _make_minimal_vault(tmp_path)
_add_literature_note(vault, "KEY00001")
import io as _io
import sys as _sys
captured = _io.StringIO()
old = _sys.stdout
_sys.stdout = captured
try:
result = _run_ocr_redo(vault, keys=["KEY00001"], dry_run=True)
finally:
_sys.stdout = old
output = captured.getvalue()
assert result == 0
assert "Would redo" in output
assert "dry-run" in output.lower()
# Dry-run must not create OCR directory
ocr_dir = vault / "System/PaperForge/ocr" / "KEY00001"
assert not ocr_dir.exists(), "Dry-run should not create OCR artifacts"
def test_redo_no_keys_falls_back(self, monkeypatch) -> None:
from paperforge.commands.ocr import _run_ocr_redo
@ -701,33 +726,43 @@ class TestOcrRedoKeyed:
assert call_kwargs.get("dry_run") is False
assert call_kwargs.get("verbose") is True
def test_redo_skips_nopdf(self, tmp_path) -> None:
def test_redo_skips_nopdf(self, tmp_path, monkeypatch) -> None:
"""nopdf papers are guarded and reported as failed, not reset."""
from paperforge.commands.ocr import _run_ocr_redo
from tests.cli.test_ocr_progress_contracts import (
_make_minimal_vault,
_add_literature_note,
_mock_run_ocr,
)
_mock_run_ocr(monkeypatch)
monkeypatch.setattr("paperforge.worker.ocr.validate_ocr_meta",
lambda _p, _m: ("done", ""))
ocr_root = _ocr_path(tmp_path)
(ocr_root / "NOPDF").mkdir(parents=True)
(ocr_root / "NOPDF" / "meta.json").write_text(
json.dumps({"zotero_key": "NOPDF", "ocr_status": "nopdf"}),
encoding="utf-8",
)
(ocr_root / "KEY1").mkdir(parents=True)
(ocr_root / "KEY1" / "meta.json").write_text(
json.dumps({"zotero_key": "KEY1", "ocr_status": "done", "ocr_job_id": "job-123"}),
vault = _make_minimal_vault(tmp_path)
# Paper with nopdf status
_add_literature_note(vault, "NOPDF001")
nopdf_meta = vault / "System/PaperForge/ocr" / "NOPDF001" / "meta.json"
nopdf_meta.parent.mkdir(parents=True, exist_ok=True)
nopdf_meta.write_text(
json.dumps({"zotero_key": "NOPDF001", "ocr_status": "nopdf"}),
encoding="utf-8",
)
# Normal redoable paper
_add_literature_note(vault, "KEY00001")
_run_ocr_redo(tmp_path, keys=["NOPDF", "KEY1"])
result = _run_ocr_redo(vault, keys=["NOPDF001", "KEY00001"])
# Should be non-zero (nopdf counts as failed)
assert result != 0
nopdf_meta = json.loads(
(ocr_root / "NOPDF" / "meta.json").read_text(encoding="utf-8")
)
assert nopdf_meta["ocr_status"] == "nopdf"
# nopdf paper must be untouched
assert nopdf_meta.exists()
import json as _json
nopdf_reloaded = _json.loads(nopdf_meta.read_text(encoding="utf-8"))
assert nopdf_reloaded["ocr_status"] == "nopdf"
key1_meta = json.loads(
(ocr_root / "KEY1" / "meta.json").read_text(encoding="utf-8")
)
assert key1_meta["ocr_status"] == "pending"
# Regular paper should have been processed
key1_meta = vault / "System/PaperForge/ocr" / "KEY00001" / "meta.json"
assert key1_meta.exists()
class TestRunOcrListDispatch: