From 1cefd4ea61cc253e1251c6a0cb01fe70bccbab9b Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 22 Mar 2026 22:44:53 +0000 Subject: [PATCH] feat: add JSONL/NDJSON file format support with sidebar preview feat: support for JSONL and NDJSON --- .changeset/evil-loops-feel.md | 5 + docs/.vitepress/config.mts | 1 + docs/data-sources/jsonl.md | 65 +++++++++ src/modules/settings/SQLSealSettingsTab.ts | 2 + src/modules/settings/init.ts | 9 +- .../SettingsJsonlControls.ts | 69 +++++++++ src/modules/settings/view/JsonlView.ts | 133 ++++++++++++++++++ .../syncStrategy/JsonlFileSyncStrategy.ts | 64 +++++++++ .../sync/syncStrategy/SyncStrategyFactory.ts | 4 + 9 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 .changeset/evil-loops-feel.md create mode 100644 docs/data-sources/jsonl.md create mode 100644 src/modules/settings/settingsTabSection/SettingsJsonlControls.ts create mode 100644 src/modules/settings/view/JsonlView.ts create mode 100644 src/modules/sync/syncStrategy/JsonlFileSyncStrategy.ts diff --git a/.changeset/evil-loops-feel.md b/.changeset/evil-loops-feel.md new file mode 100644 index 0000000..a7c30a9 --- /dev/null +++ b/.changeset/evil-loops-feel.md @@ -0,0 +1,5 @@ +--- +"sqlseal": minor +--- + +JSONL and NDJSON support diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 4159b46..cc5c0ae 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -51,6 +51,7 @@ export default defineConfig({ { text: 'Vault Data', link: '/data-sources/vault-data' }, { text: 'CSV', link: '/data-sources/csv' }, { text: 'JSON and JSON5', link: '/data-sources/json-and-json5' }, + { text: 'JSONL / NDJSON', link: '/data-sources/jsonl' }, { text: 'Markdown Tables', link: '/data-sources/markdown-tables' }, ] }, diff --git a/docs/data-sources/jsonl.md b/docs/data-sources/jsonl.md new file mode 100644 index 0000000..3c3d254 --- /dev/null +++ b/docs/data-sources/jsonl.md @@ -0,0 +1,65 @@ +# Data Source: JSONL / NDJSON + +JSONL (JSON Lines) is a format where each line is a self-contained JSON object. It is commonly used for data exports, log files, and streaming datasets. + +Both `.jsonl` and `.ndjson` (newline-delimited JSON) extensions are supported — they use the same format. + +> **Note:** SQLSeal uses the JSON5 parser for JSONL files, which means you can use JSON5 syntax (trailing commas, comments, unquoted keys, etc.) in your JSONL files. Effectively, this makes it **JSON5L** (JSON5 Lines) support. + +**Example JSONL file:** +``` +{"name": "Alice", "age": 30, "role": "admin"} +{"name": "Bob", "age": 25} +{"name": "Charlie", "age": 35, "role": "viewer"} +``` + +## Basic usage + +``` +TABLE data = file(path.jsonl) +``` + +`data` is the table alias you use in your queries. `path.jsonl` is the path to your file within your vault. + +```sqlseal +TABLE data = file(logs.jsonl) + +SELECT name, role FROM data WHERE age > 25 +``` + +## Sparse schemas + +Different lines in a JSONL file can have different keys. SQLSeal automatically collects the union of all keys across all lines and uses that as the table schema. Missing fields for a given row are stored as `NULL`. + +Given this file: +``` +{"id": 1, "event": "login", "user": "alice"} +{"id": 2, "event": "purchase", "user": "bob", "amount": 49.99} +{"id": 3, "event": "logout", "user": "alice"} +``` + +The resulting table will have columns `id`, `event`, `user`, and `amount`. Rows without `amount` will have `NULL` in that column. + +```sqlseal +TABLE events = file(events.jsonl) + +SELECT event, user, amount FROM events WHERE amount IS NOT NULL +``` + +## Nested values + +If a line contains nested objects or arrays as values, they are stored as JSON strings. You can use SQLite's `json_extract()` function to query into them. + +``` +{"id": 1, "tags": ["alpha", "beta"], "meta": {"source": "api"}} +``` + +```sqlseal +TABLE items = file(items.jsonl) + +SELECT id, json_extract(meta, '$.source') AS source FROM items +``` + +## Error handling + +Malformed lines are silently skipped with a warning in the developer console — the rest of the file loads normally. This is intentional: real-world JSONL files (especially logs) occasionally contain truncated or corrupt lines. diff --git a/src/modules/settings/SQLSealSettingsTab.ts b/src/modules/settings/SQLSealSettingsTab.ts index 59e845d..c18b1f6 100644 --- a/src/modules/settings/SQLSealSettingsTab.ts +++ b/src/modules/settings/SQLSealSettingsTab.ts @@ -7,6 +7,7 @@ export interface SQLSealSettings { enableViewer: boolean; enableEditing: boolean; enableJSONViewer: boolean; + enableJSONLViewer: boolean; enableSQLViewer: boolean; enableDynamicUpdates: boolean; enableSyntaxHighlighting: boolean; @@ -18,6 +19,7 @@ export const DEFAULT_SETTINGS: SQLSealSettings = { enableViewer: true, enableEditing: true, enableJSONViewer: true, + enableJSONLViewer: true, enableSQLViewer: true, enableDynamicUpdates: true, enableSyntaxHighlighting: true, diff --git a/src/modules/settings/init.ts b/src/modules/settings/init.ts index 229911e..e2065c6 100644 --- a/src/modules/settings/init.ts +++ b/src/modules/settings/init.ts @@ -3,6 +3,7 @@ import { SQLSealSettingsTab } from "./SQLSealSettingsTab"; import { Settings } from "./Settings"; import { SettingsCSVControls } from "./settingsTabSection/SettingsCSVControls"; import { SettingsJsonControls } from "./settingsTabSection/SettingsJsonControls"; +import { SettingsJsonlControls } from "./settingsTabSection/SettingsJsonlControls"; import { SettingsSQLControls } from "./settingsTabSection/SettingsSQLControls"; import { ViewPluginGeneratorType } from "../syntaxHighlight/viewPluginGenerator"; @@ -26,9 +27,15 @@ export const settingsInit = ( plugin, viewPluginGenerator, ); + const jsonlControl = new SettingsJsonlControls( + settings, + app, + plugin, + viewPluginGenerator, + ); const sqlControl = new SettingsSQLControls(settings, app, plugin); - const controls = [csvControl, jsonControl, sqlControl]; + const controls = [csvControl, jsonControl, jsonlControl, sqlControl]; settingsTab.registerControls(...controls); diff --git a/src/modules/settings/settingsTabSection/SettingsJsonlControls.ts b/src/modules/settings/settingsTabSection/SettingsJsonlControls.ts new file mode 100644 index 0000000..ff3a004 --- /dev/null +++ b/src/modules/settings/settingsTabSection/SettingsJsonlControls.ts @@ -0,0 +1,69 @@ +import { App, Plugin, Setting } from "obsidian"; +import { Settings } from "../Settings"; +import { SettingsControls } from "./SettingsControls"; +import { checkTypeViewAvaiability } from "../utils/viewInspector"; +import { + JSONL_VIEW_EXTENSIONS, + JSONL_VIEW_TYPE, + JsonlView, +} from "../view/JsonlView"; +import { ViewPluginGeneratorType } from "../../syntaxHighlight/viewPluginGenerator"; + +export class SettingsJsonlControls extends SettingsControls { + private registeredView: string | null = null; + + constructor(settings: Settings, app: App, plugin: Plugin, private viewPluginGenerator: ViewPluginGeneratorType) { + super(settings, app, plugin); + } + + register() { + if (this.settings.get("enableJSONLViewer")) { + const view = checkTypeViewAvaiability(this.app, JSONL_VIEW_EXTENSIONS[0]); + if (view) { + this.registeredView = view; + return; + } + + this.plugin.registerView(JSONL_VIEW_TYPE, (leaf) => new JsonlView(leaf, this.viewPluginGenerator)); + this.plugin.registerExtensions(JSONL_VIEW_EXTENSIONS, JSONL_VIEW_TYPE); + } + } + + unregister() { + this.app.workspace.detachLeavesOfType(JSONL_VIEW_TYPE); + (this.app as any).viewRegistry.unregisterExtensions([...JSONL_VIEW_EXTENSIONS]); + (this.app as any).viewRegistry.unregisterView(JSONL_VIEW_TYPE); + } + + display(el: HTMLDivElement) { + el.empty(); + + el.createEl("h2", { text: "JSONL / NDJSON File Viewer" }); + + const view = checkTypeViewAvaiability(this.app, JSONL_VIEW_EXTENSIONS[0]); + + if (view && view !== JSONL_VIEW_TYPE) { + el.createDiv({ + text: "JSONL files are already handled by a different plugin. To enable SQLSeal JSONL preview, disable the other plugin that handles it.", + cls: 'sqlseal-settings-warn' + }); + return; + } + + new Setting(el) + .setName("Enable JSONL Viewer") + .setDesc("Enables JSONL and NDJSON files in your files explorer and allows previewing them as a table.") + .addToggle((toggle) => + toggle + .setValue(this.settings.get("enableJSONLViewer")) + .onChange(async (value) => { + this.settings.set("enableJSONLViewer", !!value); + if (!!value) { + this.register(); + } else { + this.unregister(); + } + }), + ); + } +} diff --git a/src/modules/settings/view/JsonlView.ts b/src/modules/settings/view/JsonlView.ts new file mode 100644 index 0000000..f793b51 --- /dev/null +++ b/src/modules/settings/view/JsonlView.ts @@ -0,0 +1,133 @@ +import { WorkspaceLeaf, TextFileView, IconName, ButtonComponent } from 'obsidian'; +import { uniq } from 'lodash'; +import { CodeSampleModal } from '../modal/showCodeSample'; +import { ViewPluginGeneratorType } from '../../syntaxHighlight/viewPluginGenerator'; + +export const JSONL_VIEW_TYPE = "sqlseal-jsonl-viewer"; +export const JSONL_VIEW_EXTENSIONS = ['jsonl', 'ndjson']; + +const LARGE_FILE_THRESHOLD = 1000000; +const MAX_ROWS = 100; + +function parseJsonlLines(content: string): { data: Record[]; errors: number } { + const lines = content.split('\n'); + const data: Record[] = []; + let errors = 0; + + for (let i = 0; i < lines.length; i++) { + const line = (i === 0 ? lines[i].replace(/^\uFEFF/, '') : lines[i]).trim(); + if (!line) continue; + try { + const parsed = JSON.parse(line); + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + errors++; + continue; + } + data.push(parsed); + } catch { + errors++; + } + } + + return { data, errors }; +} + +export class JsonlView extends TextFileView { + private content: string = ''; + + constructor(leaf: WorkspaceLeaf, private readonly viewPluginGenerator: ViewPluginGeneratorType) { + super(leaf); + } + + getViewType(): string { + return JSONL_VIEW_TYPE; + } + + getDisplayText(): string { + return this.file?.basename || 'JSONL Viewer'; + } + + getIcon(): IconName { + return 'file-json'; + } + + async setViewData(data: string, clear: boolean): Promise { + if (clear) this.clear(); + this.content = data; + this.render(); + } + + getViewData(): string { + return this.content; + } + + clear(): void { + this.content = ''; + this.contentEl.empty(); + } + + private render(): void { + this.contentEl.empty(); + + if (!this.content) return; + + const isLarge = this.content.length > LARGE_FILE_THRESHOLD; + const container = this.contentEl.createDiv({ cls: 'sqlseal-json-viewer' }); + + if (isLarge) { + container.createDiv({ + cls: 'sqlseal-file-warning', + text: `⚠️ Large file (~${(this.content.length / 1000000).toFixed(1)}MB). Showing first ${MAX_ROWS} rows.` + }); + } + + const buttonContainer = container.createDiv({ cls: 'sqlseal-button-container' }); + const buttonComponent = new ButtonComponent(buttonContainer); + buttonComponent + .setButtonText('Generate SQLSeal Code') + .onClick(() => { + if (this.file) { + new CodeSampleModal(this.app, this.file, this.viewPluginGenerator).open(); + } + }); + + const { data, errors } = parseJsonlLines(this.content); + + if (data.length === 0) { + container.createDiv({ cls: 'sqlseal-info', text: 'No valid JSON objects found.' }); + return; + } + + const displayRows = data.slice(0, MAX_ROWS); + const columns = uniq(displayRows.slice(0, 20).flatMap(row => Object.keys(row))); + + const table = container.createEl('table', { cls: 'sqlseal-preview-table' }); + const thead = table.createEl('thead'); + const headerRow = thead.createEl('tr'); + columns.forEach(col => headerRow.createEl('th', { text: col })); + + const tbody = table.createEl('tbody'); + displayRows.forEach(row => { + const tr = tbody.createEl('tr'); + columns.forEach(col => { + const td = tr.createEl('td'); + const value = row[col]; + if (value === null || value === undefined) { + td.addClass('sqlseal-null-cell'); + } else if (typeof value === 'object') { + td.textContent = JSON.stringify(value); + td.addClass('sqlseal-object-cell'); + } else { + td.textContent = String(value); + } + }); + }); + + const total = data.length; + let infoText = total > MAX_ROWS + ? `Showing first ${MAX_ROWS} of ${total} rows (limited for performance)` + : `${total} rows`; + if (errors > 0) infoText += ` · ${errors} line(s) skipped`; + container.createDiv({ cls: 'sqlseal-row-info', text: infoText }); + } +} diff --git a/src/modules/sync/syncStrategy/JsonlFileSyncStrategy.ts b/src/modules/sync/syncStrategy/JsonlFileSyncStrategy.ts new file mode 100644 index 0000000..fa13d36 --- /dev/null +++ b/src/modules/sync/syncStrategy/JsonlFileSyncStrategy.ts @@ -0,0 +1,64 @@ +import { App } from "obsidian"; +import { ISyncStrategy } from "./abstractSyncStrategy"; +import { ParserTableDefinition } from "./types"; +import { parse } from 'json5'; +import { uniq } from "lodash"; +import { FilepathHasher } from "../../../utils/hasher"; + +const DEFAULT_FILE_HASH = '' + +export class JsonlFileSyncStrategy extends ISyncStrategy { + + static async fromParser(def: ParserTableDefinition, app: App): Promise { + const tableSourceFile = def.arguments[0] + + const path = app.metadataCache.getFirstLinkpathDest(tableSourceFile, def.sourceFile) + if (!path) { + throw new Error(`File not found: ${tableSourceFile}`) + } + const sourcePath = path.path + const hash = await FilepathHasher.sha256(sourcePath) + const tableName = `file_${hash}` + + return new JsonlFileSyncStrategy({ + arguments: def.arguments, + file_hash: DEFAULT_FILE_HASH, + refresh_id: tableName, + source_file: sourcePath, + table_name: tableName, + type: def.type + }, app) + } + + async returnData() { + const file = this.app.vault.getFileByPath(this.def.source_file)! + const fileData = await this.app.vault.cachedRead(file) + + const lines = fileData.split('\n') + const data: Record[] = [] + + for (let i = 0; i < lines.length; i++) { + // Strip BOM from first line and trim whitespace/CRLF + const line = (i === 0 ? lines[i].replace(/^\uFEFF/, '') : lines[i]).trim() + + if (!line) { + continue + } + + try { + const parsed = parse(line) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + console.warn(`SQLSeal JSONL: skipping line ${i + 1} — expected a JSON object`) + continue + } + data.push(parsed) + } catch (e) { + console.warn(`SQLSeal JSONL: skipping malformed line ${i + 1}:`, e) + } + } + + const columns = uniq(data.flatMap(d => Object.keys(d))).map(c => ({ name: c, type: 'auto' as const })) + + return { columns, data } + } +} diff --git a/src/modules/sync/syncStrategy/SyncStrategyFactory.ts b/src/modules/sync/syncStrategy/SyncStrategyFactory.ts index f748ce0..07631cc 100644 --- a/src/modules/sync/syncStrategy/SyncStrategyFactory.ts +++ b/src/modules/sync/syncStrategy/SyncStrategyFactory.ts @@ -3,6 +3,7 @@ import { ISyncStrategy } from "./abstractSyncStrategy"; import { CsvFileSyncStrategy } from "./CsvFileSyncStrategy"; import { MarkdownTableSyncStrategy } from "./MarkdownTableSyncStrategy"; import { JsonFileSyncStrategy } from "./JSONFileSyncStrategy"; +import { JsonlFileSyncStrategy } from "./JsonlFileSyncStrategy"; import { ParserTableDefinition } from "./types"; import { TableDefinitionExternal } from "../repository/tableDefinitions"; import { getFileExtension } from "../../../utils/extractExtension"; @@ -17,6 +18,9 @@ const resolveFileStrategy = (filename: string) => { case 'json': case 'json5': return JsonFileSyncStrategy + case 'jsonl': + case 'ndjson': + return JsonlFileSyncStrategy default: throw new Error(`No file processor for extension ${extension}`) }