From f7780266fae9e2dcbb83fede451bfa998968d2f3 Mon Sep 17 00:00:00 2001 From: banisterious Date: Sat, 10 Jan 2026 01:34:28 -0800 Subject: [PATCH] Fix MyHeritage GEDCOM whitespace-only line handling The preprocessor was incorrectly treating whitespace-only lines (e.g., tab-only lines) as valid GEDCOM line groups. When followed by continuation content, this produced invalid output like " [text]" instead of appending to the previous valid GEDCOM line. Changes: - Skip whitespace-only lines entirely in normalizeTabContinuations() - Add BOM stripping and warning for malformed early lines in anonymize script - Update planning doc with edge case documentation --- .../myheritage-gedcom-compatibility.md | 220 +++++++++++- src/gedcom/gedcom-preprocessor.ts | 324 +++++++++++++++++- tools/anonymize_gedcom.py | 15 +- 3 files changed, 550 insertions(+), 9 deletions(-) diff --git a/docs/planning/myheritage-gedcom-compatibility.md b/docs/planning/myheritage-gedcom-compatibility.md index 6bdf370a..36121dec 100644 --- a/docs/planning/myheritage-gedcom-compatibility.md +++ b/docs/planning/myheritage-gedcom-compatibility.md @@ -1,10 +1,17 @@ # MyHeritage GEDCOM Import Compatibility **GitHub Issue:** [#144](https://github.com/banisterious/obsidian-charted-roots/issues/144) -**Status:** Ready for Implementation +**Status:** Phase 2 Implemented **Priority:** Medium **Labels:** `accepted`, `enhancement`, `gedcom` -**Updated:** 2026-01-07 (open questions resolved with sample data from @wilbry) +**Updated:** 2026-01-10 (Phase 2: whitespace-only line handling fix) + +## Implementation Status + +| Phase | Status | Version | Description | +|-------|--------|---------|-------------| +| Phase 1 | ✅ Complete | v0.18.28-29 | BOM removal, double-encoded entities, `
` → space | +| Phase 2 | ✅ Complete | v0.19.1+ | Tab-prefixed and unprefixed continuation lines, whitespace-only line fix | ## Problem Summary @@ -427,7 +434,214 @@ if (importResult.preprocessingApplied) { --- -### Phase 2: Enhanced Reporting & Validation (Future) +### Phase 2: Tab-Prefixed Continuation Lines (NEW) + +**Discovered:** 2026-01-09 via anonymized sample from @wilbry (800KB, 42,191 lines) + +#### 2.0 Problem Analysis + +MyHeritage exports contain **non-standard continuation lines** that don't follow GEDCOM format. Instead of proper `CONC` tags for each continuation, MyHeritage produces: + +**Observed (malformed):** +```gedcom +3 TEXT [some text] +4 CONC [first continuation] + [second continuation - TAB PREFIX, NO LEVEL NUMBER] + [third continuation - TAB PREFIX, NO LEVEL NUMBER] +4 CONC [fourth continuation] +[fifth continuation - NO PREFIX AT ALL] +``` + +**Expected (per GEDCOM spec):** +```gedcom +3 TEXT [some text] +4 CONC [first continuation] +4 CONC [second continuation] +4 CONC [third continuation] +4 CONC [fourth continuation] +4 CONC [fifth continuation] +``` + +**Impact on anonymized sample:** +- **1,008 lines** start with a tab character instead of a level number +- **~1,130 lines** start with plain text (no level number, no tab) +- **Total: 2,138 malformed lines** out of 42,191 (5% of file) + +These lines cause the parser to throw "Invalid GEDCOM line format" because the regex `^(\d+)\s+(@[^@]+@\s+)?(\S+)(\s+(.*))?$` requires every line to start with a digit. + +#### 2.1 Proposed Fix + +Extend the preprocessor to normalize these continuation lines **before** parsing. + +**Strategy A: Append to Previous Line (RECOMMENDED)** + +When a line starts with tab or has no level number, append its content to the previous line: + +```typescript +function normalizeTabContinuations(content: string): { content: string; fixCount: number } { + const lines = content.split(/\r?\n/); + const normalized: string[] = []; + let fixCount = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trimStart(); + + // Check if line starts with a digit (valid GEDCOM) or is empty + if (/^\d/.test(trimmed) || !trimmed) { + normalized.push(line); + continue; + } + + // Line doesn't start with digit - it's a continuation + // Check if it starts with tab (MyHeritage continuation pattern) + if (line.startsWith('\t') || !line.match(/^\d/)) { + if (normalized.length > 0) { + // Append to previous line (with space separator) + const prevIndex = normalized.length - 1; + normalized[prevIndex] = normalized[prevIndex] + ' ' + trimmed; + fixCount++; + } else { + // Edge case: first line is malformed, keep it + normalized.push(line); + } + } + } + + return { content: normalized.join('\n'), fixCount }; +} +``` + +**Strategy B: Convert to Proper CONC Lines (Alternative)** + +Insert proper CONC tags based on the previous line's level: + +```typescript +// More complex, requires tracking level context +// May be needed if Strategy A breaks any data +``` + +**Decision: Use Strategy A** - Simpler, and CONC lines are already being joined by existing normalization. Appending to the previous line achieves the same result with less complexity. + +#### 2.2 Integration Point + +Add `normalizeTabContinuations()` as the first step in `preprocessGedcom()`, before `normalizeConcFields()`: + +```typescript +export function preprocessGedcom( + content: string, + mode: GedcomCompatibilityMode +): PreprocessResult { + // ... detection logic ... + + // Apply fixes + let processed = content; + + // Fix 0: Strip UTF-8 BOM (existing) + const bomResult = stripUtf8Bom(processed); + processed = bomResult.content; + + // Fix 1: Normalize tab-prefixed continuations (NEW) + const tabResult = normalizeTabContinuations(processed); + processed = tabResult.content; + + // Fix 2: Normalize CONC fields and repair HTML entities (existing) + const concResult = normalizeConcFields(processed); + processed = concResult.content; + + return { + content: processed, + wasPreprocessed: true, + fixes: { + bomRemoved: bomResult.fixed, + tabContinuationsFixed: tabResult.fixCount, // NEW + concFieldsNormalized: concResult.fixCount, + detectionInfo: detection + } + }; +} +``` + +#### 2.3 Detection Enhancement + +Update `detectMyHeritage()` to also check for tab-prefixed lines: + +```typescript +function detectMyHeritage(content: string): PreprocessorDetection { + // ... existing checks ... + + // Check for tab-prefixed continuation lines (sample first 50KB) + const hasTabContinuations = /^\t[^\t\n]/m.test(sample); + + return { + hasUtf8Bom, + isMyHeritage, + hasDoubleEncodedEntities, + hasTabContinuations, // NEW + shouldPreprocess: hasUtf8Bom || hasTabContinuations || + (isMyHeritage && hasDoubleEncodedEntities) + }; +} +``` + +#### 2.4 UI Update + +Update import results to report tab continuation fixes: + +```typescript +if (fixes.tabContinuationsFixed > 0) { + details.createEl('li', { + text: `Fixed ${fixes.tabContinuationsFixed} malformed continuation lines` + }); +} +``` + +#### 2.5 Testing + +**Unit tests:** +```typescript +describe('Tab Continuation Normalization', () => { + test('appends tab-prefixed lines to previous line', () => { + const input = '3 TEXT Hello\n\tworld\n\tagain'; + const result = normalizeTabContinuations(input); + expect(result.content).toBe('3 TEXT Hello world again'); + expect(result.fixCount).toBe(2); + }); + + test('handles lines with no prefix', () => { + const input = '4 CONC First\nSecond\nThird'; + const result = normalizeTabContinuations(input); + expect(result.content).toBe('4 CONC First Second Third'); + expect(result.fixCount).toBe(2); + }); + + test('preserves valid GEDCOM lines', () => { + const input = '0 @I1@ INDI\n1 NAME John /Smith/'; + const result = normalizeTabContinuations(input); + expect(result.content).toBe(input); + expect(result.fixCount).toBe(0); + }); +}); +``` + +**Integration test with anonymized sample:** +- Load `anonymized_file.ged` +- Verify no "Invalid GEDCOM line format" errors +- Verify expected number of persons/families imported + +#### 2.6 Edge Cases + +1. **First line malformed:** The anonymized sample has a malformed first line (anonymization artifact). Real MyHeritage files have `0 HEAD`. The fix should handle this gracefully - if the first line isn't valid, skip it or treat the whole line as content. + +2. **Empty lines:** Preserve empty lines as-is (they're valid in GEDCOM). + +3. **Lines with only whitespace:** ~~Trim and check - if empty after trim, preserve as empty line.~~ **Updated 2026-01-10:** Whitespace-only lines must be **skipped entirely**, not preserved. MyHeritage exports contain tab-only lines (e.g., `\t`) between continuation fragments. If these are preserved as new groups, subsequent continuation lines get appended to them, producing invalid output like `" [text]"` (space + content) instead of being appended to the previous valid GEDCOM line. + +4. **Mixed tabs and spaces:** The anonymized sample shows tab-only prefixes. If real files use mixed whitespace, `trimStart()` handles both. + +--- + +### Phase 3: Enhanced Reporting & Validation (Future) #### 2.1 Detailed Fix Log diff --git a/src/gedcom/gedcom-preprocessor.ts b/src/gedcom/gedcom-preprocessor.ts index f1914c65..3989a1d6 100644 --- a/src/gedcom/gedcom-preprocessor.ts +++ b/src/gedcom/gedcom-preprocessor.ts @@ -22,6 +22,7 @@ export interface PreprocessorDetection { hasUtf8Bom: boolean; isMyHeritage: boolean; hasDoubleEncodedEntities: boolean; + hasTabContinuations: boolean; shouldPreprocess: boolean; } @@ -33,6 +34,8 @@ export interface PreprocessResult { wasPreprocessed: boolean; fixes: { bomRemoved: boolean; + tabContinuationsFixed: number; + skippedLeadingLines: number; concFieldsNormalized: number; detectionInfo: PreprocessorDetection; }; @@ -62,11 +65,17 @@ export function detectMyHeritage(content: string): PreprocessorDetection { const sample = content.substring(0, 50000); const hasDoubleEncodedEntities = /&(?:lt|gt|amp|quot|nbsp);/i.test(sample); + // Check for tab-prefixed continuation lines (MyHeritage non-standard format) + // These are lines that start with a tab instead of a level number + const hasTabContinuations = /^\t[^\t\n]/m.test(sample); + return { hasUtf8Bom, isMyHeritage, hasDoubleEncodedEntities, - shouldPreprocess: hasUtf8Bom || (isMyHeritage && hasDoubleEncodedEntities) + hasTabContinuations, + shouldPreprocess: hasUtf8Bom || hasTabContinuations || + (isMyHeritage && hasDoubleEncodedEntities) }; } @@ -89,6 +98,127 @@ export function stripUtf8Bom(content: string): { content: string; fixed: boolean return { content, fixed: false }; } +/** + * Normalize tab-prefixed and unprefixed continuation lines + * + * MyHeritage exports non-standard GEDCOM where continuation data appears on lines + * that start with a tab character or have no prefix at all, instead of proper CONC tags. + * + * Example malformed input: + * 3 TEXT Some text + * 4 CONC first part + * second part (tab-prefixed) + * third part (tab-prefixed) + * fourth part (no prefix) + * + * This function appends such lines to the previous line with a space separator. + * Leading malformed lines (before first valid GEDCOM line) are skipped. + */ +export function normalizeTabContinuations(content: string): { content: string; fixCount: number; skippedLeadingLines: number } { + const lines = content.split(/\r?\n/); + // Use array of arrays to avoid repeated string concatenation (performance) + // Each entry is an array of parts that will be joined with space at the end + const normalized: string[][] = []; + let fixCount = 0; + let skippedLeadingLines = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trimStart(); + + // Valid GEDCOM line starts with a digit - start a new group + if (/^\d/.test(trimmed)) { + normalized.push([line]); + continue; + } + + // Empty or whitespace-only lines: skip them entirely + // In valid GEDCOM, these shouldn't exist. In MyHeritage exports, + // they appear as tab-only lines between continuation fragments. + // Skipping them allows the next continuation line to be properly + // appended to the previous valid GEDCOM line. + if (!trimmed) { + continue; + } + + // Line doesn't start with digit - it's a malformed continuation + // Append to previous line group + if (normalized.length > 0) { + normalized[normalized.length - 1].push(trimmed); + fixCount++; + } else { + // Edge case: leading malformed lines before first valid GEDCOM line + // Skip these (they're likely anonymization artifacts or file corruption) + skippedLeadingLines++; + } + } + + // Join each line group with spaces, then join all lines with newlines + return { + content: normalized.map(parts => parts.join(' ')).join('\n'), + fixCount, + skippedLeadingLines + }; +} + +/** + * Yield to the event loop to prevent UI freezing. + * Uses setTimeout with 0ms which defers to the next event loop iteration, + * allowing pending UI updates and user interactions to be processed. + */ +async function yieldToEventLoop(): Promise { + // In Electron/Obsidian, setTimeout(0) is more reliable than requestAnimationFrame + // because rAF is tied to rendering frames which may not apply in this context + return new Promise(resolve => setTimeout(resolve, 0)); +} + +/** + * Async version of normalizeTabContinuations that yields periodically. + */ +export async function normalizeTabContinuationsAsync(content: string): Promise<{ content: string; fixCount: number; skippedLeadingLines: number }> { + // Yield before the expensive split operation + await yieldToEventLoop(); + const lines = content.split(/\r?\n/); + await yieldToEventLoop(); + + const normalized: string[][] = []; + let fixCount = 0; + let skippedLeadingLines = 0; + const YIELD_INTERVAL = 5000; // Balance responsiveness with performance + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trimStart(); + + // Valid GEDCOM line starts with a digit - start a new group + if (/^\d/.test(trimmed)) { + normalized.push([line]); + } else if (!trimmed) { + // Empty or whitespace-only lines: skip them entirely + // (see sync version for explanation) + // Just continue to next line + } else if (normalized.length > 0) { + // Malformed continuation - append to previous line group + normalized[normalized.length - 1].push(trimmed); + fixCount++; + } else { + // Edge case: leading malformed lines before first valid GEDCOM line + // Skip these (they're likely anonymization artifacts or file corruption) + skippedLeadingLines++; + } + + if (i > 0 && i % YIELD_INTERVAL === 0) { + await yieldToEventLoop(); + } + } + + return { + content: normalized.map(parts => parts.join(' ')).join('\n'), + fixCount, + skippedLeadingLines + }; +} + /** * Decode common HTML entities to their character equivalents */ @@ -249,6 +379,81 @@ export function normalizeConcFields(content: string): { content: string; fixCoun }; } +/** + * Async version of normalizeConcFields that yields periodically. + */ +export async function normalizeConcFieldsAsync(content: string): Promise<{ content: string; fixCount: number }> { + // Yield before the expensive split operation + await yieldToEventLoop(); + const lines = content.split(/\r?\n/); + await yieldToEventLoop(); + + const fixed: string[] = []; + let fixCount = 0; + let i = 0; + const YIELD_INTERVAL = 5000; // Balance responsiveness with performance + let yieldCounter = 0; + + while (i < lines.length) { + const line = lines[i]; + + if (i + 1 < lines.length && /^\s*\d+\s+CONC\s/.test(lines[i + 1])) { + const levelMatch = line.match(/^\s*(\d+)/); + if (levelMatch) { + const baseLevel = parseInt(levelMatch[1]); + const concLevel = baseLevel + 1; + const concRegex = new RegExp(`^\\s*${concLevel}\\s+CONC\\s?(.*)$`); + + let accumulated = line; + let j = i + 1; + + while (j < lines.length && concRegex.test(lines[j])) { + const match = lines[j].match(concRegex); + if (match) { + accumulated += match[1]; + } + j++; + } + + const processed = repairHtmlEntities(accumulated); + if (processed.fixed) { + fixCount++; + } + fixed.push(processed.content); + + i = j; + yieldCounter += j - i; + if (yieldCounter >= YIELD_INTERVAL) { + yieldCounter = 0; + await yieldToEventLoop(); + } + continue; + } + } + + if (/&|<|>| |
= YIELD_INTERVAL) { + yieldCounter = 0; + await yieldToEventLoop(); + } + } + + return { + content: fixed.join('\n'), + fixCount + }; +} + /** * Main preprocessing function * Orchestrates all fixes based on compatibility mode setting @@ -265,11 +470,14 @@ export function preprocessGedcom( wasPreprocessed: false, fixes: { bomRemoved: false, + tabContinuationsFixed: 0, + skippedLeadingLines: 0, concFieldsNormalized: 0, detectionInfo: { hasUtf8Bom: false, isMyHeritage: false, hasDoubleEncodedEntities: false, + hasTabContinuations: false, shouldPreprocess: false } } @@ -293,6 +501,8 @@ export function preprocessGedcom( wasPreprocessed: false, fixes: { bomRemoved: false, + tabContinuationsFixed: 0, + skippedLeadingLines: 0, concFieldsNormalized: 0, detectionInfo: detection } @@ -303,7 +513,8 @@ export function preprocessGedcom( mode, isMyHeritage: detection.isMyHeritage, hasUtf8Bom: detection.hasUtf8Bom, - hasDoubleEncodedEntities: detection.hasDoubleEncodedEntities + hasDoubleEncodedEntities: detection.hasDoubleEncodedEntities, + hasTabContinuations: detection.hasTabContinuations }); // Apply fixes @@ -313,12 +524,19 @@ export function preprocessGedcom( const bomResult = stripUtf8Bom(processed); processed = bomResult.content; - // Fix 2: Normalize CONC fields and repair HTML entities + // Fix 2: Normalize tab-prefixed and unprefixed continuation lines + // Must run BEFORE normalizeConcFields so that malformed lines are joined first + const tabResult = normalizeTabContinuations(processed); + processed = tabResult.content; + + // Fix 3: Normalize CONC fields and repair HTML entities const concResult = normalizeConcFields(processed); processed = concResult.content; logger.info('preprocessGedcom', 'Preprocessing complete', { bomRemoved: bomResult.fixed, + tabContinuationsFixed: tabResult.fixCount, + skippedLeadingLines: tabResult.skippedLeadingLines, concFieldsNormalized: concResult.fixCount }); @@ -327,6 +545,106 @@ export function preprocessGedcom( wasPreprocessed: true, fixes: { bomRemoved: bomResult.fixed, + tabContinuationsFixed: tabResult.fixCount, + skippedLeadingLines: tabResult.skippedLeadingLines, + concFieldsNormalized: concResult.fixCount, + detectionInfo: detection + } + }; +} + +/** + * Async version of preprocessGedcom that yields periodically. + * Use this for large files to prevent UI freezing. + */ +export async function preprocessGedcomAsync( + content: string, + mode: GedcomCompatibilityMode +): Promise { + + // Skip if disabled + if (mode === 'none') { + return { + content, + wasPreprocessed: false, + fixes: { + bomRemoved: false, + tabContinuationsFixed: 0, + skippedLeadingLines: 0, + concFieldsNormalized: 0, + detectionInfo: { + hasUtf8Bom: false, + isMyHeritage: false, + hasDoubleEncodedEntities: false, + hasTabContinuations: false, + shouldPreprocess: false + } + } + }; + } + + // Detect issues + const detection = detectMyHeritage(content); + + // Should we preprocess? + const shouldPreprocess = mode === 'myheritage' || + (mode === 'auto' && detection.shouldPreprocess); + + if (!shouldPreprocess) { + logger.debug('preprocessGedcomAsync', 'No preprocessing needed', { + mode, + detection + }); + return { + content, + wasPreprocessed: false, + fixes: { + bomRemoved: false, + tabContinuationsFixed: 0, + skippedLeadingLines: 0, + concFieldsNormalized: 0, + detectionInfo: detection + } + }; + } + + logger.info('preprocessGedcomAsync', 'Applying MyHeritage compatibility fixes', { + mode, + isMyHeritage: detection.isMyHeritage, + hasUtf8Bom: detection.hasUtf8Bom, + hasDoubleEncodedEntities: detection.hasDoubleEncodedEntities, + hasTabContinuations: detection.hasTabContinuations + }); + + // Apply fixes + let processed = content; + + // Fix 1: Strip UTF-8 BOM (synchronous, fast) + const bomResult = stripUtf8Bom(processed); + processed = bomResult.content; + + // Fix 2: Normalize tab-prefixed and unprefixed continuation lines (async) + const tabResult = await normalizeTabContinuationsAsync(processed); + processed = tabResult.content; + + // Fix 3: Normalize CONC fields and repair HTML entities (async) + const concResult = await normalizeConcFieldsAsync(processed); + processed = concResult.content; + + logger.info('preprocessGedcomAsync', 'Preprocessing complete', { + bomRemoved: bomResult.fixed, + tabContinuationsFixed: tabResult.fixCount, + skippedLeadingLines: tabResult.skippedLeadingLines, + concFieldsNormalized: concResult.fixCount + }); + + return { + content: processed, + wasPreprocessed: true, + fixes: { + bomRemoved: bomResult.fixed, + tabContinuationsFixed: tabResult.fixCount, + skippedLeadingLines: tabResult.skippedLeadingLines, concFieldsNormalized: concResult.fixCount, detectionInfo: detection } diff --git a/tools/anonymize_gedcom.py b/tools/anonymize_gedcom.py index 24b87385..2ed536db 100644 --- a/tools/anonymize_gedcom.py +++ b/tools/anonymize_gedcom.py @@ -18,6 +18,7 @@ What gets anonymized: What gets preserved: - GEDCOM structure and syntax + - Header (0 HEAD) and trailer (0 TRLR) records - Record relationships (family links, parent-child connections) - Record types (INDI, FAM, SOUR, etc.) - Tag structure and hierarchy @@ -77,11 +78,19 @@ class GedcomAnonymizer: else: return date # Unknown format, keep as-is - def anonymize_line(self, line: str) -> str: + def anonymize_line(self, line: str, line_number: int = 0) -> str: """Anonymize a single GEDCOM line.""" + # Strip BOM if present (can appear if file encoding wasn't handled correctly) + if line.startswith('\ufeff'): + line = line[1:] + # Extract level, tag, and value match = re.match(r'^(\d+)\s+(@[^@]+@\s+)?(\w+)(\s+(.*))?$', line) if not match: + # Non-matching lines are kept as-is (likely malformed continuations) + # But warn if it's one of the first few lines (might be a header issue) + if line_number < 5 and line.strip(): + print(f" Warning: Line {line_number + 1} doesn't match GEDCOM format: {line[:50]!r}") return line level, xref, tag, _, value = match.groups() @@ -155,11 +164,11 @@ class GedcomAnonymizer: print(f"Processing {len(lines)} lines...") anonymized_lines = [] - for line in lines: + for i, line in enumerate(lines): # Strip line ending but preserve structure line = line.rstrip('\n\r') if line.strip(): - anonymized_line = self.anonymize_line(line) + anonymized_line = self.anonymize_line(line, line_number=i) anonymized_lines.append(anonymized_line) else: anonymized_lines.append(line)