From 077c190b606d4ac28918736816ce28fa0e3cf7c0 Mon Sep 17 00:00:00 2001 From: LLLin000 <809867916@qq.com> Date: Thu, 9 Jul 2026 00:30:50 +0800 Subject: [PATCH] docs: update project management for PR9A-C embedding pipeline work --- PROJECT-MANAGEMENT.md | 8 +- paperforge/plugin/src/settings.ts | 158 ++++++++++++++++++------- project/current/ocr-v2-active-queue.md | 95 ++++++++------- 3 files changed, 169 insertions(+), 92 deletions(-) diff --git a/PROJECT-MANAGEMENT.md b/PROJECT-MANAGEMENT.md index 9710b1b2..ffd13a12 100644 --- a/PROJECT-MANAGEMENT.md +++ b/PROJECT-MANAGEMENT.md @@ -1,11 +1,9 @@ -# OCR-v2 Project Management Log - -> **Branch:** `master` | **Last Updated:** 2026-07-05 -> **Active work:** Layer 3 — Plugin UI polish. Layer 2 (OCR Quality Report + Readiness Policy) delivered at `96fd9771`. Full test suite: 1278 passed, 8 pre-existing failures, 275 skipped. +> **Branch:** `master` | **Last Updated:** 2026-07-08 +> **Active work:** Embedding pipeline overhaul (PR9A-C). Previous: Layer 3 — Plugin UI polish. Layer 2 (OCR Quality Report + Readiness Policy) delivered at `96fd9771`. Full Python test suite: 41 passed (14 PR9A + 23 PR9B + 4 PR9C), plugin tests: 58 passed. > > --- > -> **Current state:** All four architecture-hardening PRs (frontmatter fallback, caption heuristic, column-aware zone, pairing column) merged. V3 normalize split ON by default. Layer 2 (Quality Indicators + Readiness Policy + Feedback Sidecar) implemented with contract polish complete. Next: Plugin UI polish (Layer 3), then downstream tooling (Layer 4). Architecture cleanup deferred post-release. +> **Current state:** Memory/Embedding layer rearchitected. PR9A (correctness) — resume/rebuild deterministic selection, `.done` markers removed, three-gate embed resume protection. PR9B (performance) — three-stage pipeline (prepare/encode/write) with parallel encode via ThreadPoolExecutor. PR9C (UX) — sliding-window streaming pipeline with continuous EMBED_PROGRESS. Provider switched from `openai` client to `requests` to fix SiliconFlow NAT hang. Plugin chunk display fixed to sum all three collections. Next: full vault embed build with new pipeline. ## 1. Architecture diff --git a/paperforge/plugin/src/settings.ts b/paperforge/plugin/src/settings.ts index 808d7d03..1c29ec16 100644 --- a/paperforge/plugin/src/settings.ts +++ b/paperforge/plugin/src/settings.ts @@ -1220,8 +1220,12 @@ export class PaperForgeSettingTab extends PluginSettingTab { }); } else { const embedInfo = getVectorRuntime(vp); - const hasChunks = !!(embedInfo && (embedInfo.chunk_count ?? 0) > 0); - const isCorrupted = embedInfo && (embedInfo as any).corrupted; + const embedChunks = + (embedInfo?.chunk_count ?? 0) + + (embedInfo?.body_chunk_count ?? 0) + + (embedInfo?.object_chunk_count ?? 0); + const hasChunks = embedChunks > 0; + const isCorrupted = embedInfo ? !!embedInfo.corrupted : false; const startBuild = (flag: string) => { const py = getCachedPython(vp, this.plugin.settings); @@ -1313,7 +1317,7 @@ export class PaperForgeSettingTab extends PluginSettingTab { if (hasChunks && !isCorrupted) { embedControls.createEl("span", { - text: embedInfo.chunk_count + " chunks embedded", + text: embedChunks + " chunks embedded", cls: "setting-item-description", }); } @@ -1720,9 +1724,15 @@ export class PaperForgeSettingTab extends PluginSettingTab { // ── Helpers ── const buildActionArgs = (keys: string[], action: string) => { if (action === "retry_ocr" || action === "upgrade_legacy") - return { cmd: ["-m", "paperforge", "ocr", "redo", ...keys], timeout: 300000 }; + return { + cmd: ["-m", "paperforge", "ocr", "redo", ...keys], + timeout: 300000, + }; if (action === "rebuild_result") - return { cmd: ["-m", "paperforge", "ocr", "rebuild", ...keys], timeout: 120000 }; + return { + cmd: ["-m", "paperforge", "ocr", "rebuild", ...keys], + timeout: 120000, + }; return null; }; @@ -1751,9 +1761,11 @@ export class PaperForgeSettingTab extends PluginSettingTab { const renderTable = (papers: MaintenanceDisplayRow[]) => { statusEl.empty(); - const visible = papers.filter(p => p.visible_in_maintenance); + const visible = papers.filter((p) => p.visible_in_maintenance); if (visible.length === 0) { - statusEl.createEl("p", { text: t("maintenance_all_good") || "✅ 全部正常" }); + statusEl.createEl("p", { + text: t("maintenance_all_good") || "✅ 全部正常", + }); return; } @@ -1761,13 +1773,29 @@ export class PaperForgeSettingTab extends PluginSettingTab { const pyExtra = (py.extraArgs || []) as string[]; // Group by display_group - const groups: { key: string; title: string; items: MaintenanceDisplayRow[] }[] = [ - { key: "retry", title: t("maintenance_group_retry") || "需要重试", items: [] }, - { key: "rebuild", title: t("maintenance_group_rebuild") || "可重建结果", items: [] }, - { key: "legacy_optional", title: t("maintenance_group_legacy") || "可升级旧结果（可选）", items: [] }, + const groups: { + key: string; + title: string; + items: MaintenanceDisplayRow[]; + }[] = [ + { + key: "retry", + title: t("maintenance_group_retry") || "需要重试", + items: [], + }, + { + key: "rebuild", + title: t("maintenance_group_rebuild") || "可重建结果", + items: [], + }, + { + key: "legacy_optional", + title: t("maintenance_group_legacy") || "可升级旧结果（可选）", + items: [], + }, ]; for (const p of visible) { - const g = groups.find(g => g.key === p.display_group); + const g = groups.find((g) => g.key === p.display_group); if (g) g.items.push(p); } @@ -1781,9 +1809,13 @@ export class PaperForgeSettingTab extends PluginSettingTab { if (isLegacy) { const summary = (section as HTMLDetailsElement).createEl("summary"); - summary.createEl("strong", { text: group.title + " (" + group.items.length + ")" }); + summary.createEl("strong", { + text: group.title + " (" + group.items.length + ")", + }); } else { - section.createEl("h3", { text: group.title + " (" + group.items.length + ")" }); + section.createEl("h3", { + text: group.title + " (" + group.items.length + ")", + }); } // Selection state per row @@ -1794,11 +1826,16 @@ export class PaperForgeSettingTab extends PluginSettingTab { const toolbar = section.createEl("div", { cls: "pf-maint-toolbar" }); const selAllBtn = toolbar.createEl("button", { text: "全选" }); const deselAllBtn = toolbar.createEl("button", { text: "取消全选" }); - const execBtn = toolbar.createEl("button", { text: "▶ 执行已选", cls: "mod-cta" }); - const execLabel = toolbar.createEl("span", { cls: "pf-maint-exec-label" }); + const execBtn = toolbar.createEl("button", { + text: "▶ 执行已选", + cls: "mod-cta", + }); + const execLabel = toolbar.createEl("span", { + cls: "pf-maint-exec-label", + }); const updateExecLabel = () => { - const n = group.items.filter(p => selState.get(p.key)).length; + const n = group.items.filter((p) => selState.get(p.key)).length; execLabel.setText("已选 " + n + " 篇"); }; updateExecLabel(); @@ -1807,18 +1844,26 @@ export class PaperForgeSettingTab extends PluginSettingTab { for (const p of group.items) selState.set(p.key, true); updateExecLabel(); // Update checkboxes - const cbs = section.querySelectorAll("input[type=checkbox].pf-maint-sel"); - cbs.forEach((cb) => { (cb as HTMLInputElement).checked = true; }); + const cbs = section.querySelectorAll( + "input[type=checkbox].pf-maint-sel" + ); + cbs.forEach((cb) => { + (cb as HTMLInputElement).checked = true; + }); }); deselAllBtn.addEventListener("click", () => { for (const p of group.items) selState.set(p.key, false); updateExecLabel(); - const cbs = section.querySelectorAll("input[type=checkbox].pf-maint-sel"); - cbs.forEach((cb) => { (cb as HTMLInputElement).checked = false; }); + const cbs = section.querySelectorAll( + "input[type=checkbox].pf-maint-sel" + ); + cbs.forEach((cb) => { + (cb as HTMLInputElement).checked = false; + }); }); execBtn.addEventListener("click", () => { - const selected = group.items.filter(p => selState.get(p.key)); + const selected = group.items.filter((p) => selState.get(p.key)); if (selected.length === 0) { new Notice("请先选择要处理的论文。"); return; @@ -1843,16 +1888,19 @@ export class PaperForgeSettingTab extends PluginSettingTab { const thead = table.createEl("thead"); const tbody = table.createEl("tbody"); const headerRow = thead.insertRow(); - ["", "Key", "Title", "建议操作", "原因", "操作"].forEach(h => { + ["", "Key", "Title", "建议操作", "原因", "操作"].forEach((h) => { const th = document.createElement("th"); th.textContent = h; headerRow.appendChild(th); }); const btnText = (action: string) => { - if (action === "retry_ocr") return t("maintenance_btn_retry") || "重试"; - if (action === "rebuild_result") return t("maintenance_btn_rebuild") || "重建"; - if (action === "upgrade_legacy") return t("maintenance_btn_upgrade") || "升级"; + if (action === "retry_ocr") + return t("maintenance_btn_retry") || "重试"; + if (action === "rebuild_result") + return t("maintenance_btn_rebuild") || "重建"; + if (action === "upgrade_legacy") + return t("maintenance_btn_upgrade") || "升级"; return ""; }; @@ -1874,12 +1922,14 @@ export class PaperForgeSettingTab extends PluginSettingTab { // Key const keyTd = tr.insertCell(); - keyTd.style.cssText = "padding:3px 4px;white-space:nowrap;font-size:11px;max-width:90px;overflow:hidden;text-overflow:ellipsis;"; + keyTd.style.cssText = + "padding:3px 4px;white-space:nowrap;font-size:11px;max-width:90px;overflow:hidden;text-overflow:ellipsis;"; keyTd.textContent = p.key; // Title const titleTd = tr.insertCell(); - titleTd.style.cssText = "padding:3px 4px;white-space:nowrap;max-width:220px;overflow:hidden;text-overflow:ellipsis;"; + titleTd.style.cssText = + "padding:3px 4px;white-space:nowrap;max-width:220px;overflow:hidden;text-overflow:ellipsis;"; titleTd.textContent = p.title || p.key; // Display label (建议操作) @@ -1889,12 +1939,14 @@ export class PaperForgeSettingTab extends PluginSettingTab { // Reason const reasonTd = tr.insertCell(); - reasonTd.style.cssText = "padding:3px 4px;white-space:nowrap;max-width:160px;overflow:hidden;text-overflow:ellipsis;font-size:11px;color:var(--text-muted);"; + reasonTd.style.cssText = + "padding:3px 4px;white-space:nowrap;max-width:160px;overflow:hidden;text-overflow:ellipsis;font-size:11px;color:var(--text-muted);"; reasonTd.textContent = p.display_reason || ""; // Action button const actionTd = tr.insertCell(); - actionTd.style.cssText = "padding:3px 4px;text-align:center;white-space:nowrap;"; + actionTd.style.cssText = + "padding:3px 4px;text-align:center;white-space:nowrap;"; const actionBtn = document.createElement("button"); actionBtn.textContent = btnText(p.display_action); if (actionBtn.textContent) { @@ -1905,7 +1957,9 @@ export class PaperForgeSettingTab extends PluginSettingTab { pyPath, [...pyExtra, ...args.cmd], { cwd: vaultPath, timeout: args.timeout, windowsHide: true }, - () => { new Notice(p.display_label + " — " + p.key); } + () => { + new Notice(p.display_label + " — " + p.key); + } ); }); actionTd.appendChild(actionBtn); @@ -1923,20 +1977,25 @@ export class PaperForgeSettingTab extends PluginSettingTab { } // ── Phase 2: Background refresh ── - refreshMaintenanceData(vaultPath, py.path, py.extraArgs || [], cache || null) - .then(result => { + refreshMaintenanceData( + vaultPath, + py.path, + py.extraArgs || [], + cache || null + ) + .then((result) => { if (result.changed) { renderTable(result.data); writeMaintenanceCache(vaultPath, { manifest: {}, - papers: Object.fromEntries(result.data.map(p => [p.key, p])), + papers: Object.fromEntries(result.data.map((p) => [p.key, p])), cached_at: new Date().toISOString(), }); } else if (!cache) { renderTable(result.data); writeMaintenanceCache(vaultPath, { manifest: {}, - papers: Object.fromEntries(result.data.map(p => [p.key, p])), + papers: Object.fromEntries(result.data.map((p) => [p.key, p])), cached_at: new Date().toISOString(), }); } @@ -1954,27 +2013,44 @@ export class PaperForgeSettingTab extends PluginSettingTab { // ── Global operations ── containerEl.createEl("hr"); containerEl.createEl("h3", { text: "全局操作" }); - const globalActions = containerEl.createEl("div", { cls: "pf-maint-global" }); + const globalActions = containerEl.createEl("div", { + cls: "pf-maint-global", + }); - const rebuildIndexBtn = globalActions.createEl("button", { text: "重建搜索索引" }); + const rebuildIndexBtn = globalActions.createEl("button", { + text: "重建搜索索引", + }); rebuildIndexBtn.addEventListener("click", () => { new Notice("正在重建搜索索引…"); execFile( py.path, - [...(py.extraArgs || []), "-m", "paperforge", "embed", "build", "--force"], + [ + ...(py.extraArgs || []), + "-m", + "paperforge", + "embed", + "build", + "--force", + ], { cwd: vaultPath, timeout: 300000, windowsHide: true }, - () => { new Notice("搜索索引重建完成。"); } + () => { + new Notice("搜索索引重建完成。"); + } ); }); - const rebuildMemBtn = globalActions.createEl("button", { text: "重建记忆库" }); + const rebuildMemBtn = globalActions.createEl("button", { + text: "重建记忆库", + }); rebuildMemBtn.addEventListener("click", () => { new Notice("正在重建记忆库…"); execFile( py.path, [...(py.extraArgs || []), "-m", "paperforge", "repair", "--fix"], { cwd: vaultPath, timeout: 120000, windowsHide: true }, - () => { new Notice("记忆库重建完成。"); } + () => { + new Notice("记忆库重建完成。"); + } ); }); } diff --git a/project/current/ocr-v2-active-queue.md b/project/current/ocr-v2-active-queue.md index 94ab87e0..30d0f81c 100644 --- a/project/current/ocr-v2-active-queue.md +++ b/project/current/ocr-v2-active-queue.md @@ -1,50 +1,53 @@ # OCR-v2 Active Queue -> Status: ACTIVE QUEUE — Four-layer release readiness. Layer 2 delivered. 1278 tests green on `master`. -> Last updated: 2026-07-05 -> Scope: UI polish → downstream tools +> Status: Embedding pipeline overhaul complete. PR9A-C all merged to `master`. +> Last updated: 2026-07-08 -## Current Priorities -1. ✅ **Workstream X: Layout-category truth audit** — 11 papers, 6 bug patterns. Report in `docs/superpowers/analysis/2026-07-05-layout-truth-audit-findings.md` -2. ✅ **Layer 2: OCR Quality Report + Readiness Policy** — `build_quality_indicators()`, `evaluate_readiness()`, human feedback sidecar. Contract polished at commit `96fd9771`. -3. 🟡 **Plugin UI polish** — dashboard cleanliness, maintenance display -4. 🟡 **Downstream tooling** — section-aware vector chunking, figure/table separate handling +## Completed (this session) -## Completed This Session (cumulative) -- **Workstream A — object writeback seam**: - - Added `paperforge/worker/ocr_object_writeback.py` - - Unified figure/table asset writeback, contained text, side-adjacent text, and consumed-block ownership evidence - - Added `tests/test_ocr_object_writeback.py` -- **Workstream B — tail settlement seam**: - - Added `paperforge/worker/ocr_tail_settlement.py` - - Added `TailSettlementReport` and attached it to `DocumentStructure` - - Preserved legacy tail/body/backmatter behavior with focused regressions -- **Workstream C — v3 pre/post normalize split**: - - Added `paperforge/worker/ocr_pre_match_normalize.py` - - Added `paperforge/worker/ocr_post_match_normalize.py` - - Added `OCR_PIPELINE_V3` toggle and `normalize_mode="seed_only"` - - Updated figure/table matching to accept `role_candidate > role > seed_role` -- **Pre-merge blocker cleanup**: - - Fixed page-qualified object writeback lookup - - Guarded contained figure claims by page - - Restored rescue equivalence inside `post_match_normalize()` - - Added merge-gate regression tests for all four blockers -- **Verification**: - - `tests/test_ocr_pipeline_v3.py` - - `tests/test_ocr_tail_settlement.py` - - `tests/test_ocr_object_writeback.py` - - `tests/test_appendix_figure_numbering.py` - - `tests/test_ocr_rendering.py` - - Result: **105 passed, 0 failed** -- **Layer 2: OCR Quality Report + Readiness Policy**: - - Added `paperforge/worker/ocr_quality.py` — `build_quality_indicators()` with 5 normalizers - - Added `paperforge/worker/ocr_quality_feedback.py` — human feedback sidecar (per-mark hash, stale detection) - - Added `paperforge/policies/ocr_readiness_v1.yaml` — default readiness policy (weights, hard-red, use-case gates) - - Added `evaluate_readiness()` — policy evaluator with deep-merge, user override bypass - - Contract polish: `status/gates/reasons` output shape, hash validation, non-mutating append - - 22 new tests (17 quality + 5 feedback), 1278 total green - - Master commit: `96fd9771` +### PR9A: Resume & Rebuild Correctness +- OCR rebuild `--all`: version/artifact-based selection via `_needs_derived_rebuild()` +- `--status` and explicit keys: manual override, no version filter +- `.done.{key}` checkpoint markers removed from selection +- `_apply_post_rebuild_version_flags` now writes `derived_version` +- Embed resume entry: three-gate protection (stale state / missing DB / corrupt DB) +- `_pid_alive()` cross-platform PID health check +- 14 regression tests -## Immediate Next Checks -- [ ] Send 6 bug patterns to GPT for solution design -- [ ] Fix 37LK5T97 two-column figure 1 bug (Round 2 RED) -- [ ] Archive stale queue docs from pairing-framework phase +### PR9B: Embed Parallel Encode +- Four dataclasses: `EmbeddingPayload`, `EncodedPayload`, `PaperEmbeddingJob`, `PaperEncodedBundle` +- `prepare_legacy/body/object_payload` — prepare phase extraction +- `encode_payload` / `encode_paper_job` — worker-thread-safe encode +- `write_encoded_payload` — ChromaDB serial write +- Existing `embed_body_units`/`embed_paper` refactored as wrappers +- 23 regression tests + +### Provider Fix +- Switched `OpenAICompatibleProvider` from `openai` client to `requests` +- Fixed SiliconFlow NAT connection hang (openai-python#3269) +- 0.3s vs 2.3s init, no more hanging + +### PR9C: Streaming Embed Pipeline +- Sliding-window pipeline: prepare + submit bounded in-flight papers +- `wait(FIRST_COMPLETED)` in main thread for encode results +- `processed_count` = skip + embedded, monotonic EMBED_PROGRESS +- Resume skip and no-payload paths also advance processed_count +- Encode failure fails closed (return 1, no silent skip) +- `write_vector_build_state` fallback when file locked (Windows) +- 4 integration tests + +### Plugin Fixes +- Status text shows total chunks across all three collections +- "chunks embedded" text below progress bar also uses total + +## Test Status +| Suite | Result | +|-------|--------| +| Python unit tests (PR9A) | 14/14 pass | +| Python unit tests (PR9B) | 23/23 pass | +| Python unit tests (PR9C) | 4/4 pass | +| Plugin tests | 58/58 pass | +| Full vault embed build | 729/729 papers, 20,655 chunks | + +## Immediate Next +- [ ] Full vault embed build (`--force`) on fresh ChromaDB +- [ ] Verify chunk counts in Obsidian plugin display