feat: add JSONL/NDJSON file format support with sidebar preview (#203)

feat: support for JSONL and NDJSON

.changeset/evil-loops-feel.md

@@ -0,0 +1,5 @@
+---
+"sqlseal": minor
+---
+
+JSONL and NDJSON support

docs/.vitepress/config.mts

@@ -51,6 +51,7 @@ export default defineConfig({
           { text: 'Vault Data', link: '/data-sources/vault-data' },
           { text: 'CSV', link: '/data-sources/csv' },
           { text: 'JSON and JSON5', link: '/data-sources/json-and-json5' },
+          { text: 'JSONL / NDJSON', link: '/data-sources/jsonl' },
           { text: 'Markdown Tables', link: '/data-sources/markdown-tables' },
         ]
       },

docs/data-sources/jsonl.md

@@ -0,0 +1,65 @@
+# Data Source: JSONL / NDJSON
+
+JSONL (JSON Lines) is a format where each line is a self-contained JSON object. It is commonly used for data exports, log files, and streaming datasets.
+
+Both `.jsonl` and `.ndjson` (newline-delimited JSON) extensions are supported — they use the same format.
+
+> **Note:** SQLSeal uses the JSON5 parser for JSONL files, which means you can use JSON5 syntax (trailing commas, comments, unquoted keys, etc.) in your JSONL files. Effectively, this makes it **JSON5L** (JSON5 Lines) support.
+
+**Example JSONL file:**
+```
+{"name": "Alice", "age": 30, "role": "admin"}
+{"name": "Bob", "age": 25}
+{"name": "Charlie", "age": 35, "role": "viewer"}
+```
+
+## Basic usage
+
+```
+TABLE data = file(path.jsonl)
+```
+
+`data` is the table alias you use in your queries. `path.jsonl` is the path to your file within your vault.
+
+```sqlseal
+TABLE data = file(logs.jsonl)
+
+SELECT name, role FROM data WHERE age > 25
+```
+
+## Sparse schemas
+
+Different lines in a JSONL file can have different keys. SQLSeal automatically collects the union of all keys across all lines and uses that as the table schema. Missing fields for a given row are stored as `NULL`.
+
+Given this file:
+```
+{"id": 1, "event": "login", "user": "alice"}
+{"id": 2, "event": "purchase", "user": "bob", "amount": 49.99}
+{"id": 3, "event": "logout", "user": "alice"}
+```
+
+The resulting table will have columns `id`, `event`, `user`, and `amount`. Rows without `amount` will have `NULL` in that column.
+
+```sqlseal
+TABLE events = file(events.jsonl)
+
+SELECT event, user, amount FROM events WHERE amount IS NOT NULL
+```
+
+## Nested values
+
+If a line contains nested objects or arrays as values, they are stored as JSON strings. You can use SQLite's `json_extract()` function to query into them.
+
+```
+{"id": 1, "tags": ["alpha", "beta"], "meta": {"source": "api"}}
+```
+
+```sqlseal
+TABLE items = file(items.jsonl)
+
+SELECT id, json_extract(meta, '$.source') AS source FROM items
+```
+
+## Error handling
+
+Malformed lines are silently skipped with a warning in the developer console — the rest of the file loads normally. This is intentional: real-world JSONL files (especially logs) occasionally contain truncated or corrupt lines.

src/modules/settings/SQLSealSettingsTab.ts

@@ -7,6 +7,7 @@ export interface SQLSealSettings {
     enableViewer: boolean;
     enableEditing: boolean;
     enableJSONViewer: boolean;
+    enableJSONLViewer: boolean;
     enableSQLViewer: boolean;
     enableDynamicUpdates: boolean;
     enableSyntaxHighlighting: boolean;
@@ -18,6 +19,7 @@ export const DEFAULT_SETTINGS: SQLSealSettings = {
     enableViewer: true,
     enableEditing: true,
     enableJSONViewer: true,
+    enableJSONLViewer: true,
     enableSQLViewer: true,
     enableDynamicUpdates: true,
     enableSyntaxHighlighting: true,

src/modules/settings/init.ts

@@ -3,6 +3,7 @@ import { SQLSealSettingsTab } from "./SQLSealSettingsTab";
 import { Settings } from "./Settings";
 import { SettingsCSVControls } from "./settingsTabSection/SettingsCSVControls";
 import { SettingsJsonControls } from "./settingsTabSection/SettingsJsonControls";
+import { SettingsJsonlControls } from "./settingsTabSection/SettingsJsonlControls";
 import { SettingsSQLControls } from "./settingsTabSection/SettingsSQLControls";
 import { ViewPluginGeneratorType } from "../syntaxHighlight/viewPluginGenerator";
 
@@ -26,9 +27,15 @@ export const settingsInit = (
 			plugin,
 			viewPluginGenerator,
 		);
+		const jsonlControl = new SettingsJsonlControls(
+			settings,
+			app,
+			plugin,
+			viewPluginGenerator,
+		);
 		const sqlControl = new SettingsSQLControls(settings, app, plugin);
 
-		const controls = [csvControl, jsonControl, sqlControl];
+		const controls = [csvControl, jsonControl, jsonlControl, sqlControl];
 
 		settingsTab.registerControls(...controls);
 

src/modules/settings/settingsTabSection/SettingsJsonlControls.ts

@@ -0,0 +1,69 @@
+import { App, Plugin, Setting } from "obsidian";
+import { Settings } from "../Settings";
+import { SettingsControls } from "./SettingsControls";
+import { checkTypeViewAvaiability } from "../utils/viewInspector";
+import {
+	JSONL_VIEW_EXTENSIONS,
+	JSONL_VIEW_TYPE,
+	JsonlView,
+} from "../view/JsonlView";
+import { ViewPluginGeneratorType } from "../../syntaxHighlight/viewPluginGenerator";
+
+export class SettingsJsonlControls extends SettingsControls {
+	private registeredView: string | null = null;
+
+	constructor(settings: Settings, app: App, plugin: Plugin, private viewPluginGenerator: ViewPluginGeneratorType) {
+		super(settings, app, plugin);
+	}
+
+	register() {
+		if (this.settings.get("enableJSONLViewer")) {
+			const view = checkTypeViewAvaiability(this.app, JSONL_VIEW_EXTENSIONS[0]);
+			if (view) {
+				this.registeredView = view;
+				return;
+			}
+
+			this.plugin.registerView(JSONL_VIEW_TYPE, (leaf) => new JsonlView(leaf, this.viewPluginGenerator));
+			this.plugin.registerExtensions(JSONL_VIEW_EXTENSIONS, JSONL_VIEW_TYPE);
+		}
+	}
+
+	unregister() {
+		this.app.workspace.detachLeavesOfType(JSONL_VIEW_TYPE);
+		(this.app as any).viewRegistry.unregisterExtensions([...JSONL_VIEW_EXTENSIONS]);
+		(this.app as any).viewRegistry.unregisterView(JSONL_VIEW_TYPE);
+	}
+
+	display(el: HTMLDivElement) {
+		el.empty();
+
+		el.createEl("h2", { text: "JSONL / NDJSON File Viewer" });
+
+		const view = checkTypeViewAvaiability(this.app, JSONL_VIEW_EXTENSIONS[0]);
+
+		if (view && view !== JSONL_VIEW_TYPE) {
+			el.createDiv({
+				text: "JSONL files are already handled by a different plugin. To enable SQLSeal JSONL preview, disable the other plugin that handles it.",
+				cls: 'sqlseal-settings-warn'
+			});
+			return;
+		}
+
+		new Setting(el)
+			.setName("Enable JSONL Viewer")
+			.setDesc("Enables JSONL and NDJSON files in your files explorer and allows previewing them as a table.")
+			.addToggle((toggle) =>
+				toggle
+					.setValue(this.settings.get("enableJSONLViewer"))
+					.onChange(async (value) => {
+						this.settings.set("enableJSONLViewer", !!value);
+						if (!!value) {
+							this.register();
+						} else {
+							this.unregister();
+						}
+					}),
+			);
+	}
+}

src/modules/settings/view/JsonlView.ts

@@ -0,0 +1,133 @@
+import { WorkspaceLeaf, TextFileView, IconName, ButtonComponent } from 'obsidian';
+import { uniq } from 'lodash';
+import { CodeSampleModal } from '../modal/showCodeSample';
+import { ViewPluginGeneratorType } from '../../syntaxHighlight/viewPluginGenerator';
+
+export const JSONL_VIEW_TYPE = "sqlseal-jsonl-viewer";
+export const JSONL_VIEW_EXTENSIONS = ['jsonl', 'ndjson'];
+
+const LARGE_FILE_THRESHOLD = 1000000;
+const MAX_ROWS = 100;
+
+function parseJsonlLines(content: string): { data: Record<string, unknown>[]; errors: number } {
+    const lines = content.split('\n');
+    const data: Record<string, unknown>[] = [];
+    let errors = 0;
+
+    for (let i = 0; i < lines.length; i++) {
+        const line = (i === 0 ? lines[i].replace(/^\uFEFF/, '') : lines[i]).trim();
+        if (!line) continue;
+        try {
+            const parsed = JSON.parse(line);
+            if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+                errors++;
+                continue;
+            }
+            data.push(parsed);
+        } catch {
+            errors++;
+        }
+    }
+
+    return { data, errors };
+}
+
+export class JsonlView extends TextFileView {
+    private content: string = '';
+
+    constructor(leaf: WorkspaceLeaf, private readonly viewPluginGenerator: ViewPluginGeneratorType) {
+        super(leaf);
+    }
+
+    getViewType(): string {
+        return JSONL_VIEW_TYPE;
+    }
+
+    getDisplayText(): string {
+        return this.file?.basename || 'JSONL Viewer';
+    }
+
+    getIcon(): IconName {
+        return 'file-json';
+    }
+
+    async setViewData(data: string, clear: boolean): Promise<void> {
+        if (clear) this.clear();
+        this.content = data;
+        this.render();
+    }
+
+    getViewData(): string {
+        return this.content;
+    }
+
+    clear(): void {
+        this.content = '';
+        this.contentEl.empty();
+    }
+
+    private render(): void {
+        this.contentEl.empty();
+
+        if (!this.content) return;
+
+        const isLarge = this.content.length > LARGE_FILE_THRESHOLD;
+        const container = this.contentEl.createDiv({ cls: 'sqlseal-json-viewer' });
+
+        if (isLarge) {
+            container.createDiv({
+                cls: 'sqlseal-file-warning',
+                text: `⚠️ Large file (~${(this.content.length / 1000000).toFixed(1)}MB). Showing first ${MAX_ROWS} rows.`
+            });
+        }
+
+        const buttonContainer = container.createDiv({ cls: 'sqlseal-button-container' });
+        const buttonComponent = new ButtonComponent(buttonContainer);
+        buttonComponent
+            .setButtonText('Generate SQLSeal Code')
+            .onClick(() => {
+                if (this.file) {
+                    new CodeSampleModal(this.app, this.file, this.viewPluginGenerator).open();
+                }
+            });
+
+        const { data, errors } = parseJsonlLines(this.content);
+
+        if (data.length === 0) {
+            container.createDiv({ cls: 'sqlseal-info', text: 'No valid JSON objects found.' });
+            return;
+        }
+
+        const displayRows = data.slice(0, MAX_ROWS);
+        const columns = uniq(displayRows.slice(0, 20).flatMap(row => Object.keys(row)));
+
+        const table = container.createEl('table', { cls: 'sqlseal-preview-table' });
+        const thead = table.createEl('thead');
+        const headerRow = thead.createEl('tr');
+        columns.forEach(col => headerRow.createEl('th', { text: col }));
+
+        const tbody = table.createEl('tbody');
+        displayRows.forEach(row => {
+            const tr = tbody.createEl('tr');
+            columns.forEach(col => {
+                const td = tr.createEl('td');
+                const value = row[col];
+                if (value === null || value === undefined) {
+                    td.addClass('sqlseal-null-cell');
+                } else if (typeof value === 'object') {
+                    td.textContent = JSON.stringify(value);
+                    td.addClass('sqlseal-object-cell');
+                } else {
+                    td.textContent = String(value);
+                }
+            });
+        });
+
+        const total = data.length;
+        let infoText = total > MAX_ROWS
+            ? `Showing first ${MAX_ROWS} of ${total} rows (limited for performance)`
+            : `${total} rows`;
+        if (errors > 0) infoText += ` · ${errors} line(s) skipped`;
+        container.createDiv({ cls: 'sqlseal-row-info', text: infoText });
+    }
+}

src/modules/sync/syncStrategy/JsonlFileSyncStrategy.ts

@@ -0,0 +1,64 @@
+import { App } from "obsidian";
+import { ISyncStrategy } from "./abstractSyncStrategy";
+import { ParserTableDefinition } from "./types";
+import { parse } from 'json5';
+import { uniq } from "lodash";
+import { FilepathHasher } from "../../../utils/hasher";
+
+const DEFAULT_FILE_HASH = ''
+
+export class JsonlFileSyncStrategy extends ISyncStrategy {
+
+    static async fromParser(def: ParserTableDefinition, app: App): Promise<JsonlFileSyncStrategy> {
+        const tableSourceFile = def.arguments[0]
+
+        const path = app.metadataCache.getFirstLinkpathDest(tableSourceFile, def.sourceFile)
+        if (!path) {
+            throw new Error(`File not found: ${tableSourceFile}`)
+        }
+        const sourcePath = path.path
+        const hash = await FilepathHasher.sha256(sourcePath)
+        const tableName = `file_${hash}`
+
+        return new JsonlFileSyncStrategy({
+            arguments: def.arguments,
+            file_hash: DEFAULT_FILE_HASH,
+            refresh_id: tableName,
+            source_file: sourcePath,
+            table_name: tableName,
+            type: def.type
+        }, app)
+    }
+
+    async returnData() {
+        const file = this.app.vault.getFileByPath(this.def.source_file)!
+        const fileData = await this.app.vault.cachedRead(file)
+
+        const lines = fileData.split('\n')
+        const data: Record<string, unknown>[] = []
+
+        for (let i = 0; i < lines.length; i++) {
+            // Strip BOM from first line and trim whitespace/CRLF
+            const line = (i === 0 ? lines[i].replace(/^\uFEFF/, '') : lines[i]).trim()
+
+            if (!line) {
+                continue
+            }
+
+            try {
+                const parsed = parse(line)
+                if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+                    console.warn(`SQLSeal JSONL: skipping line ${i + 1} — expected a JSON object`)
+                    continue
+                }
+                data.push(parsed)
+            } catch (e) {
+                console.warn(`SQLSeal JSONL: skipping malformed line ${i + 1}:`, e)
+            }
+        }
+
+        const columns = uniq(data.flatMap(d => Object.keys(d))).map(c => ({ name: c, type: 'auto' as const }))
+
+        return { columns, data }
+    }
+}

src/modules/sync/syncStrategy/SyncStrategyFactory.ts

@@ -3,6 +3,7 @@ import { ISyncStrategy } from "./abstractSyncStrategy";
 import { CsvFileSyncStrategy } from "./CsvFileSyncStrategy";
 import { MarkdownTableSyncStrategy } from "./MarkdownTableSyncStrategy";
 import { JsonFileSyncStrategy } from "./JSONFileSyncStrategy";
+import { JsonlFileSyncStrategy } from "./JsonlFileSyncStrategy";
 import { ParserTableDefinition } from "./types";
 import { TableDefinitionExternal } from "../repository/tableDefinitions";
 import { getFileExtension } from "../../../utils/extractExtension";
@@ -17,6 +18,9 @@ const resolveFileStrategy = (filename: string) => {
         case 'json':
         case 'json5':
             return JsonFileSyncStrategy
+        case 'jsonl':
+        case 'ndjson':
+            return JsonlFileSyncStrategy
         default:
             throw new Error(`No file processor for extension ${extension}`)
     }