From a35cccffcd130630be846e80e1c6f57c5aca7e40 Mon Sep 17 00:00:00 2001 From: Research Assistant Date: Mon, 1 Jun 2026 13:05:18 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20minimal-intervention=20base=20sanitizer?= =?UTF-8?q?=20=E2=80=94=20dedup,=20fix=20syntax,=20preserve=20content,=20e?= =?UTF-8?q?nsure=204=20standard=20views?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- paperforge/worker/_utils.py | 11 +- paperforge/worker/base_views.py | 213 +++++++++++++++++++++++++++++--- tests/test_base_preservation.py | 25 ++-- 3 files changed, 218 insertions(+), 31 deletions(-) diff --git a/paperforge/worker/_utils.py b/paperforge/worker/_utils.py index abb5f31f..5af6159d 100644 --- a/paperforge/worker/_utils.py +++ b/paperforge/worker/_utils.py @@ -20,12 +20,19 @@ STANDARD_VIEW_NAMES = frozenset( # --- Journal Database --- -from paperforge.core.date_utils import extract_year as _extract_year -from paperforge.core.io import read_json, write_json +from paperforge.core.io import read_json, write_json # noqa: F401 _JOURNAL_DB: dict[str, dict] | None = None +def _extract_year(pubdate: str) -> str: + """Extract a 4-digit year from a pubdate string.""" + if not pubdate: + return "" + match = re.search(r"(\d{4})", str(pubdate)) + return match.group(1) if match else "" + + def load_journal_db(vault: Path) -> dict[str, dict]: """Load zoterostyle.json journal database.""" global _JOURNAL_DB diff --git a/paperforge/worker/base_views.py b/paperforge/worker/base_views.py index a340546b..e0898206 100644 --- a/paperforge/worker/base_views.py +++ b/paperforge/worker/base_views.py @@ -421,14 +421,201 @@ views: {views_yaml}""" -def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: bool = False) -> None: - """Generate Domain Base files on first run; subsequent calls only update folder filter. +def _sanitize_base_file(content: str, views: list[dict], folder_filter: str) -> str: + """Minimally sanitize a .base file: deduplicate views, fix syntax, ensure standard views exist. - Base views are static — Obsidian automatically reflects frontmatter changes. - PaperForge should not regenerate views on every sync; it only ensures: - 1. Base files exist (create on first setup). - 2. Folder filter stays in sync with configured directory paths. - 3. force=True allows full regeneration when explicitly requested. + This function does NOT modify existing view content (columns, filters, widths, sort). + It only: + 1. Removes duplicate views with the same name (keeps PF-prefixed, then first occurrence). + 2. Fixes common syntax issues (quoted !zotero_key.isEmpty(), broken ocr_redo placement). + 3. Updates the folder filter for file.inFolder(). + 4. Ensures the 4 standard views exist (appends missing ones with PAPERFORGE_VIEW_PREFIX). + 5. Ensures ocr_redo property exists at the correct position in properties section. + 6. Other views are left completely untouched. + """ + STANDARD_NAMES = {v["name"] for v in views} + + # --- Phase 1: split into header (pre-views:) and view blocks --- + lines = content.split("\n") + views_start_idx = None + for i, line in enumerate(lines): + if line.strip().startswith("views:"): + views_start_idx = i + break + + if views_start_idx is None: + # Missing views: section — treat whole file as header, append views + header = content.rstrip("\n") + if not header.endswith("\n"): + header += "\n" + fresh_views_yaml = _render_views_section(views) + return f"{header}\nviews:\n{fresh_views_yaml}\n" + + header_lines = lines[: views_start_idx + 1] + view_body_lines = lines[views_start_idx + 1 :] + + # --- Phase 2: parse view blocks --- + # A view block is either: PAPERFORGE_VIEW_PREFIX + indented block, or - type: table + indented block + view_blocks = [] # list of (is_pf: bool, view_name: str, block_text: str) + i = 0 + while i < len(view_body_lines): + line = view_body_lines[i] + if not line.strip(): + i += 1 + continue + + if line.startswith(PAPERFORGE_VIEW_PREFIX): + view_name = line[len(PAPERFORGE_VIEW_PREFIX):].strip() + block_lines = [line] + i += 1 + type_table_count = 0 + while i < len(view_body_lines): + next_line = view_body_lines[i] + if next_line.strip().startswith(PAPERFORGE_VIEW_PREFIX): + break + if next_line.strip().startswith("- type: table"): + if type_table_count >= 1: + break + type_table_count += 1 + if next_line.strip() and not next_line.startswith(" ") and not next_line.startswith("\t"): + break + block_lines.append(next_line) + i += 1 + view_blocks.append((True, view_name, "\n".join(block_lines))) + continue + + elif line.strip().startswith("- type: table"): + block_lines = [line] + i += 1 + while i < len(view_body_lines): + next_line = view_body_lines[i] + if next_line.strip().startswith(PAPERFORGE_VIEW_PREFIX): + break + if next_line.strip().startswith("- type: table"): + break + if next_line.strip() and not next_line.startswith(" ") and not next_line.startswith("\t"): + break + block_lines.append(next_line) + i += 1 + block_text = "\n".join(block_lines) + name_match = re.search(r'name:\s*"?([^"]*)"?', block_text) + view_name = name_match.group(1) if name_match else "" + view_blocks.append((False, view_name, block_text)) + continue + else: + i += 1 + + # --- Phase 3: deduplicate --- + # For each view name, keep PF-prefixed if exists, otherwise first occurrence + deduped: dict[str, str] = {} # view_name -> block_text (NO prefix included for non-PF) + for is_pf, view_name, block_text in view_blocks: + if not view_name: + continue + if view_name in deduped: + if is_pf: + # PF block wins over user block + deduped[view_name] = block_text + # else: keep existing (first occurrence or existing PF) + else: + deduped[view_name] = block_text + + # --- Phase 4: ensure standard views exist --- + # For each standard name NOT in deduped, generate a fresh PF view block + # For existing standard views, keep them AS-IS (don't replace content) + for v in views: + if v["name"] not in deduped: + block = f"{PAPERFORGE_VIEW_PREFIX}{v['name']}\n" + block += " - type: table\n" + block += f' name: "{v["name"]}"\n' + block += " order:\n" + for col in v["order"]: + block += f" - {col}\n" + if v["filter"]: + block += f" filter: '{v['filter']}'\n" + deduped[v["name"]] = block + + # --- Phase 5: fix properties section --- + header_text = "\n".join(header_lines) + header_lines_fixed = header_text.split("\n") + + # 5a: Fix quoted !zotero_key.isEmpty() + for j in range(len(header_lines_fixed)): + stripped = header_lines_fixed[j].strip() + if stripped in ('"!zotero_key.isEmpty()"', "'!zotero_key.isEmpty()'"): + header_lines_fixed[j] = header_lines_fixed[j].replace('"!zotero_key.isEmpty()"', "!zotero_key.isEmpty()") + header_lines_fixed[j] = header_lines_fixed[j].replace("'!zotero_key.isEmpty()'", "!zotero_key.isEmpty()") + + # 5b: Fix broken ocr_redo placement (between ocr_status: and its displayName) + # Strip all existing ocr_redo blocks, then re-insert after ocr_status block + cleaned = [] + skip_next = False + for j, line in enumerate(header_lines_fixed): + if skip_next: + skip_next = False + continue + if line.strip() == "ocr_redo:" and (j + 1 < len(header_lines_fixed)): + next_stripped = header_lines_fixed[j + 1].strip() + if "displayName" in next_stripped or next_stripped.startswith("displayName"): + skip_next = True + continue + cleaned.append(line) + header_lines_fixed = cleaned + + # Find ocr_status block and insert ocr_redo after its displayName + target_idx = None + for j, line in enumerate(header_lines_fixed): + if line.strip() == "ocr_status:": + # Find the end of ocr_status block (its displayName line, if any) + if j + 1 < len(header_lines_fixed) and "displayName" in header_lines_fixed[j + 1]: + target_idx = j + 2 # After displayName + else: + target_idx = j + 1 # After ocr_status: + break + + if target_idx is not None: + has_redo = any( + line.strip() == "ocr_redo:" for line in header_lines_fixed[target_idx : target_idx + 2] + ) + if not has_redo: + header_lines_fixed.insert(target_idx, " ocr_redo:") + header_lines_fixed.insert(target_idx + 1, ' displayName: "重做OCR"') + + # 5c: Update folder filter + for j in range(len(header_lines_fixed)): + if 'file.inFolder(' in header_lines_fixed[j]: + header_lines_fixed[j] = re.sub( + r'file\.inFolder\("[^"]*"\)', + f'file.inFolder("{folder_filter}")', + header_lines_fixed[j], + ) + break + + # --- Phase 6: rebuild --- + rebuilt = "\n".join(header_lines_fixed) + "\n" + # Rebuild view blocks in order: standard views first, then non-standard (preserve original order) + view_order = [] + seen = set() + for v in views: + if v["name"] in deduped and v["name"] not in seen: + view_order.append(v["name"]) + seen.add(v["name"]) + for view_name in deduped: + if view_name not in seen: + view_order.append(view_name) + seen.add(view_name) + + for view_name in view_order: + rebuilt += deduped[view_name] + "\n" + + return rebuilt + + +def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: bool = False) -> None: + """Generate and minimally sanitize domain Base files. + + For existing files (force=False): sanitizes — dedups views, fixes syntax, + ensures 4 standard views exist. Does NOT modify existing view content. + For new files or force=True: generates fresh from template. """ paths["bases"].mkdir(parents=True, exist_ok=True) @@ -436,15 +623,9 @@ def ensure_base_views(vault: Path, paths: dict[str, Path], config: dict, force: resolved_filter = substitute_config_placeholders(folder_filter, paths) if base_path.exists() and not force: existing = base_path.read_text(encoding="utf-8") - existing = re.sub( - r'file\.inFolder\("[^"]+"\)', - f'file.inFolder("{resolved_filter}")', - existing, - count=1, - ) - merged = merge_base_views(existing, views, folder_filter=resolved_filter) - if merged != existing: - base_path.write_text(merged, encoding="utf-8") + sanitized = _sanitize_base_file(existing, views, folder_filter=resolved_filter) + if sanitized != existing: + base_path.write_text(sanitized, encoding="utf-8") return merged = _build_base_yaml(resolved_filter, views) merged = substitute_config_placeholders(merged, paths) diff --git a/tests/test_base_preservation.py b/tests/test_base_preservation.py index 681e11b8..94909b56 100644 --- a/tests/test_base_preservation.py +++ b/tests/test_base_preservation.py @@ -105,12 +105,11 @@ class TestIncrementalMerge: assert refreshed.count(PAPERFORGE_VIEW_PREFIX) == 4 def test_user_modified_filter_on_standard_view_is_preserved(self): - """User changes filter on a standard PF view — on refresh it is REPLACED. + """User changes filter on a standard PF view — on refresh it is PRESERVED. - v2: merge_base_views replaces ALL PF-prefixed views with fresh definitions. - User modifications to standard view filters are reset. - User-added views (no prefix) are preserved. - force=True does full regeneration (same behavior as before). + v3: _sanitize_base_file does NOT touch existing view content. + User modifications to standard view filters, columns, widths, sort survive refresh. + Only: duplicate removal, syntax fixes, missing standard views appended. """ domain_base = self.bases / f"{slugify_filename('骨科')}.base" ensure_base_views(self.vault, self.paths, self.config, force=False) @@ -125,10 +124,9 @@ class TestIncrementalMerge: ensure_base_views(self.vault, self.paths, self.config, force=False) refreshed = domain_base.read_text(encoding="utf-8") - # v2: merge_base_views replaces PF views — user modification should be reset - assert ocr_filter_old in refreshed, "Standard PF view should be reset by merge_base_views" - assert ocr_filter_modified not in refreshed, "User filter modification should be replaced" - # force=True should still regenerate + # v3: existing view content is preserved — user modification survives + assert ocr_filter_modified in refreshed, "User filter modification should be preserved" + # force=True should still regenerate (user mods lost when force) ensure_base_views(self.vault, self.paths, self.config, force=True) force_refreshed = domain_base.read_text(encoding="utf-8") assert ocr_filter_old in force_refreshed @@ -147,10 +145,10 @@ class TestIncrementalMerge: assert PAPERFORGE_VIEW_PREFIX in content def test_old_file_without_prefix_gets_views_without_duplication(self): - """Base files from old versions (no PAPERFORGE_VIEW_PREFIX) get views added once.""" + """Old base files (no PAPERFORGE_VIEW_PREFIX): existing views preserved, missing standard views appended.""" domain_base = self.bases / f"{slugify_filename('骨科')}.base" - # Simulate old-version file: views without PAPERFORGE_VIEW_PREFIX + # Simulate old-version file: 2 views without PAPERFORGE_VIEW_PREFIX old_content = """filters: and: - file.inFolder("Resources/Literature/骨科") @@ -175,11 +173,12 @@ views: ensure_base_views(self.vault, self.paths, self.config, force=False) content = domain_base.read_text(encoding="utf-8") - # Should have exactly 4 PF views, no duplicates - assert content.count(PAPERFORGE_VIEW_PREFIX) == 4 + # Old views preserved as-is (no prefix added to existing); missing views appended with prefix assert content.count('name: "控制面板"') == 1 assert content.count('name: "待 OCR"') == 1 + assert content.count(PAPERFORGE_VIEW_PREFIX) == 2 # Only 待深度阅读 + 重做OCR added assert "重做OCR" in content + assert "待深度阅读" in content def test_corrupted_file_with_eight_views_repaired_to_four(self): """Post-corruption: 4 old prefix-free views + 4 new PF-prefixed views = 8.