Merge pull request #97 from h-sphere/feat/define-type

feat: Ability to manually define types for columns
This commit is contained in:
Kacper Kula 2025-05-22 15:42:28 +02:00 committed by GitHub
commit d765fb7f7f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 715 additions and 471 deletions

View file

@ -0,0 +1,5 @@
---
"sqlseal": minor
---
added ability to manually define column types

View file

@ -37,6 +37,7 @@ export default defineConfig({
{ text: 'Links and Images', link: '/links-and-images' },
{ text: 'CSV Viewer', link: '/csv-viewer' },
{ text: 'Query Configuration', link: '/query-configuration' },
{ text: 'Define Column Types', link: '/define-column-types' },
{ text: 'Troubleshooting', link: '/troubleshooting' },
{ text: 'Future Plans', link: '/future-plans' },
{ text: 'Changelog', link: '/changelog' }

View file

@ -0,0 +1,7 @@
# Define Column Types (CSV files)
SQLSeal sets types of your data automatically. This is the mechanism which for most of the cases works perfectly, but for more advanced use-cases, you might want to force specific type of the column (for example you might have ids which you want to treat like text, even if they consist of just the numbers). To enforce a specific type, you can set individual types inside the CSV Viewer:
![Setting Column Type](./setting-column-type.png)
Changing data type enables corresponding controls.

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

View file

@ -31,11 +31,11 @@
"@types/estraverse": "^5.1.7",
"@types/jest": "^29.5.14",
"@types/lodash": "^4.17.16",
"@types/node": "^22.15.12",
"@types/papaparse": "^5.3.15",
"@types/node": "^22.15.18",
"@types/papaparse": "^5.3.16",
"@types/sql.js": "^1.4.9",
"@typescript-eslint/eslint-plugin": "8.32.0",
"@typescript-eslint/parser": "8.32.0",
"@typescript-eslint/eslint-plugin": "8.32.1",
"@typescript-eslint/parser": "8.32.1",
"builtin-modules": "5.0.0",
"electron-rebuild": "^3.2.9",
"esbuild": "0.25.4",
@ -43,23 +43,23 @@
"jest": "^29.7.0",
"obsidian": "^1.8.7",
"prettier": "3.5.3",
"ts-jest": "^29.3.2",
"ts-jest": "^29.3.4",
"tslib": "2.8.1",
"typescript": "5.8.3",
"vitepress": "^1.6.3",
"vue": "^3.5.13"
"vue": "^3.5.14"
},
"dependencies": {
"@ag-grid-community/theming": "^32.3.5",
"@codemirror/language": "^6.11.0",
"@codemirror/state": "^6.5.2",
"@codemirror/view": "^6.36.7",
"@codemirror/view": "^6.36.8",
"@hypersphere/omnibus": "^0.1.6",
"@jlongster/sql.js": "^1.6.7",
"@types/jsonpath": "^0.2.4",
"@vanakat/plugin-api": "^0.2.1",
"absurd-sql": "^0.0.54",
"ag-grid-community": "^33.2.4",
"ag-grid-community": "^33.3.0",
"comlink": "^4.4.2",
"esbuild-plugin-polyfill-node": "^0.3.0",
"esprima": "^4.0.1",

File diff suppressed because it is too large Load diff

View file

@ -4,6 +4,7 @@ import * as Comlink from 'comlink'
import workerCode from 'virtual:worker-code'
import { WorkerDatabase } from "./worker/database";
import { sanitise } from "../utils/sanitiseColumn";
import { ColumnDefinition } from "../utils/types";
export class SqlSealDatabase {
db: Comlink.Remote<WorkerDatabase>
@ -83,6 +84,10 @@ export class SqlSealDatabase {
await this.db.createTableNoTypes(name, columns, noDrop)
}
async createTable(name: string, columns: ColumnDefinition[], noDrop?: boolean) {
this.db.createTable(name, columns, noDrop)
}
async createIndex(indexName: string, tableName: string, columns: string[]) {
await this.db.createIndex(indexName, tableName, columns)
}

View file

@ -9,7 +9,8 @@ import { SQLiteFS } from 'absurd-sql';
import IndexedDBBackend from '../../../node_modules/absurd-sql/dist/indexeddb-backend.js';
import type { BindParams, Database, Statement } from "sql.js";
import { sanitise } from "../../utils/sanitiseColumn";
import { uniq } from "lodash";
import { uniq, uniqBy } from "lodash";
import { ColumnDefinition } from "../../utils/types";
function toObjectArray(stmt: Statement) {
@ -131,6 +132,22 @@ export class WorkerDatabase {
this.db.run(createStmt)
}
async createTable(tableName: string, columns: ColumnDefinition[], noDrop: boolean = false) {
const fields = uniqBy(columns.map(c => ({
...c,
name: sanitise(c.name)
})), 'name')
if (!noDrop) {
await this.dropTable(tableName)
}
const fieldDefinitions = fields.map(c => c.name) // Setting type inside engine changes nothing for SQLite
const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fieldDefinitions.join(', ')})`
this.db.run(createStmt)
}
async clearTable(tableName: string) {
this.db.run(`DELETE FROM ${tableName}`)

View file

@ -92,7 +92,7 @@ export class Sync {
const { data, columns } = await syncObject.returnData()
const tableName = entry.table_name
await this.db.createTableNoTypes(tableName, columns)
await this.db.createTable(tableName, columns)
await this.db.insertData(tableName, data)
await this.tableDefinitionsRepo.update(entry.id, { file_hash: file.stat.mtime.toString() })
this.bus.trigger('change::' + tableName)

View file

@ -5,6 +5,7 @@ import { FilepathHasher } from "../../utils/hasher";
import { TableDefinitionExternal } from "../repository/tableDefinitions";
import { ParserTableDefinition } from "./types";
import { App } from "obsidian";
import { loadConfig } from "../../utils/csvConfig";
const DEFAULT_FILE_HASH = ''
@ -60,12 +61,12 @@ export class CsvFileSyncStrategy extends ISyncStrategy {
skipEmptyLines: true,
transformHeader: sanitise
})
// const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
// const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
// name: key,
// type: value as FieldTypes
// }));
return { data: parsed.data, columns: parsed.meta.fields ?? [] }
const config = await loadConfig(file, this.app.vault)
return { data: parsed.data, columns: parsed.meta.fields?.map(f => ({
name: f,
type: config.columnDefinitions[f]?.type ?? 'auto' as const
})) ?? [] }
}
}

View file

@ -53,7 +53,7 @@ export class JsonFileSyncStrategy extends ISyncStrategy {
throw new Error('Resulting data is not an array')
}
const columns = uniq(data.map(d => Object.keys(d)).flat())
const columns = uniq(data.map(d => Object.keys(d)).flat()).map(c => ({ name: c, type: 'auto' as const }))
return { columns, data: data }
}

View file

@ -171,7 +171,7 @@ export class MarkdownTableSyncStrategy extends ISyncStrategy {
return {
data,
columns: headers ?? []
columns: headers.map(c => ({ name: c, type: 'auto' as const })) ?? []
};
}
}

View file

@ -1,6 +1,7 @@
import { App } from "obsidian";
import { TableDefinitionExternal } from "../repository/tableDefinitions";
import { ParserTableDefinition } from "./types";
import { ColumnDefinition } from "../../utils/types";
export abstract class ISyncStrategy {
constructor(protected def: TableDefinitionExternal, protected app: App) {
@ -11,7 +12,7 @@ export abstract class ISyncStrategy {
}
abstract returnData(): Promise<{
data: Record<string, unknown>[],
columns: string[]
columns: ColumnDefinition[]
}>;
static async fromParser(def: ParserTableDefinition, app: App): Promise<ISyncStrategy> {

View file

@ -26,13 +26,13 @@ const getAgGridTheme = (theme: 'dark' | 'light') => {
} as const
}
class GridRendererCommunicator {
export class GridRendererCommunicator {
constructor(
private el: HTMLElement,
private config: Partial<GridOptions>,
private plugin: SqlSealPlugin | null,
private app: App,
private cellParser: ModernCellParser
private cellParser?: ModernCellParser
) {
this.initialize()
this.setupLayoutObservers()
@ -105,11 +105,11 @@ class GridRendererCommunicator {
theme: myTheme,
defaultColDef: {
resizable: false,
cellRendererSelector: () => {
cellRendererSelector: this.cellParser ? () => {
return {
component: ({ value }: { value: string }) => this.cellParser.render(value)
component: ({ value }: { value: string }) => this.cellParser!.render(value)
}
},
} : undefined,
autoHeight: true
},
autoSizeStrategy: {
@ -137,10 +137,6 @@ class GridRendererCommunicator {
if (!this.gridApi) {
throw new Error('Grid has not been initiated')
}
this.gridApi.setGridOption('columnDefs', columns.map((c: any) => ({
headerName: c,
valueGetter: (params) => params.data[c]
})))
this.gridApi.setGridOption('rowData', data)
this.gridApi.setGridOption('loading', false)
}
@ -194,7 +190,8 @@ export class GridRenderer implements RendererConfig {
cleanup: () => {
communicator.cleanup()
communicator.gridApi.destroy()
}
},
communicator
}
}
}

View file

@ -57,7 +57,7 @@ export class ListRenderer implements RendererConfig {
text: createEl('span', { text: c, cls: 'sqlseal-column-name' }) as any, // FIXME: this should be properly typed
cls: singleCol ? ['sqlseal-list-element', 'sqlseal-list-element-single'] : ['sqlseal-list-element-single']
})
const val: any = cellParser.render(d[c])
const val: any = cellParser!.render(d[c])
el.append(val)
el.dataset.sqlsealColumn = c
})

View file

@ -5,7 +5,6 @@ import { RendererConfig, RendererContext } from "../renderer/rendererRegistry";
import { displayError } from "../utils/ui";
import { ViewDefinition } from "../grammar/parser";
import { ParseResults } from "../cellParser/parseResults";
import { ModernCellParser } from "../cellParser/ModernCellParser";
const mapDataFromHeaders = (columns: string[], data: Record<string, any>[]) => {
return data.map(d => columns.map(c => String(d[c])))
@ -31,7 +30,7 @@ export class MarkdownRenderer implements RendererConfig {
}
render(config: ReturnType<typeof this.validateConfig>, el: HTMLElement, { cellParser } : RendererContext) {
const parseResult = new ParseResults(cellParser)
const parseResult = new ParseResults(cellParser!)
return {
render: ({ columns, data }: any) => {
const tab = getMarkdownTable({

View file

@ -3,7 +3,6 @@ import { App } from "obsidian";
import { RendererConfig, RendererContext } from "../renderer/rendererRegistry";
import { displayError } from "../utils/ui";
import { ViewDefinition } from "../grammar/parser";
import { ModernCellParser } from "../cellParser/ModernCellParser";
interface HTMLRendererConfig {
classNames: string[]
@ -73,7 +72,7 @@ export class TableRenderer implements RendererConfig {
data.forEach((d: any) => {
const row = body.createEl("tr")
columns.forEach((c: any) => {
const parsed = cellParser.render(d[c]) as string
const parsed = cellParser!.render(d[c]) as string
if (adjustLayout) {
const td = row.createEl("td")
td.createSpan({ text: parsed })

View file

@ -42,7 +42,7 @@ export class TemplateRenderer implements RendererConfig {
render: ({ columns, data, frontmatter }: any) => {
el.empty()
const parser = new ParseResults(cellParser, (el) => new Handlebars.SafeString(el.outerHTML))
const parser = new ParseResults(cellParser!, (el) => new Handlebars.SafeString(el.outerHTML))
// Seems to be the only way to render handlebars into DOM. Don't like it but what can we do.
el.innerHTML = config.template({

View file

@ -7,7 +7,7 @@ export interface DataFormat {
}
export interface RendererContext {
cellParser: ModernCellParser,
cellParser?: ModernCellParser,
sourcePath: string
}

46
src/utils/csvConfig.ts Normal file
View file

@ -0,0 +1,46 @@
import { TFile, Vault } from "obsidian"
import { parse as jsonParse, stringify as jsonStringify } from 'json5'
// export type ColumnType = 'auto' | 'number' | 'text'
export type ColumnType = string
interface ColumnDefinition {
type: ColumnType
}
export interface ConfigObject {
columnDefinitions: {[key: string]: ColumnDefinition }
}
export const loadConfig = async (file: TFile, vault: Vault): Promise<ConfigObject> => {
const config = {
columnDefinitions: {}
}
// Now loading saved one
const configPath = file!.path + '.sqlsealconfig'
const configFile = vault.getFileByPath(configPath)
if (!configFile) {
return config
}
const loadedConfig = await vault.read(configFile)
// decode
const decoded = jsonParse(loadedConfig)
return decoded
}
export const saveConfig = async (file: TFile, content: Object, vault: Vault) => {
const serialised = jsonStringify(content, null, 2)
// Check if exists
const configPath = file!.path + '.sqlsealconfig'
const configFile = vault.getFileByPath(configPath)
if (!configFile) {
// CREATE NEW
await vault.create(configPath, serialised)
return
}
await vault.modify(configFile, serialised)
}

7
src/utils/types.ts Normal file
View file

@ -0,0 +1,7 @@
// export type ColumnType = 'auto' | 'text' | 'number'
export type ColumnType = string
export interface ColumnDefinition {
name: string;
type: ColumnType
}

View file

@ -1,11 +1,15 @@
import { WorkspaceLeaf, TextFileView, Menu, Notice } from 'obsidian';
import { WorkspaceLeaf, TextFileView, Menu, Notice, MenuItem } from 'obsidian';
import { parse, unparse } from 'papaparse';
import { DeleteConfirmationModal } from '../modal/deleteConfirmationModal';
import { RenameColumnModal } from '../modal/renameColumnModal';
import { CodeSampleModal } from '../modal/showCodeSample';
import { GridRenderer } from '../renderer/GridRenderer';
import { GridRenderer, GridRendererCommunicator } from '../renderer/GridRenderer';
import { errorNotice } from '../utils/notice';
import { ModernCellParser } from '../cellParser/ModernCellParser';
import { ConfigObject, loadConfig, saveConfig } from 'src/utils/csvConfig';
import { ColumnType } from '../utils/types';
const delay = (n: number) => new Promise(resolve => setTimeout(resolve, n))
export const CSV_VIEW_TYPE = "csv-viewer" as const;
export const CSV_VIEW_EXTENSIONS = ['csv'];
@ -13,6 +17,7 @@ export const CSV_VIEW_EXTENSIONS = ['csv'];
export class CSVView extends TextFileView {
private content: string;
private table: HTMLTableElement;
private config: ConfigObject;
constructor(
leaf: WorkspaceLeaf,
@ -60,7 +65,7 @@ export class CSVView extends TextFileView {
private result: any;
updateRow(newRowData: Record<string, any>) {
this.result.data[parseInt(newRowData.__index, 10)] = newRowData;
this.saveData()
this.saveData(true)
}
deleteRow(idx: number) {
@ -102,6 +107,7 @@ export class CSVView extends TextFileView {
return d
})
this.saveData()
this.loadDataIntoGrid()
}
setIsEditable(newValue: boolean) {
@ -109,9 +115,25 @@ export class CSVView extends TextFileView {
// FIXME: if there already rendered view, use this to rerender it?
}
saveData() {
const output = unparse(this.result)
saveData(noRefresh: boolean = false) {
if (noRefresh) {
this.refreshSkip = Date.now() + 500
}
// Map results
const res = [...this.result.data].map(r => {
const r2 = {...r}
Object.keys(r).forEach(k => {
if (typeof r[k] === 'boolean') {
r2[k] = r[k] ? 1 : 0
}
})
return r2
})
const output = unparse({ ...this.result, data: res })
this.app.vault.modify(this.file!, output)
this.refreshTypes()
}
createRow() {
@ -120,6 +142,66 @@ export class CSVView extends TextFileView {
}
getColumnType(columnName: string) {
if (this.config.columnDefinitions[columnName]) {
return this.config.columnDefinitions[columnName].type
}
return 'auto'
}
async changeColumnType(columnName: string, type: ColumnType) {
const prev = this.config.columnDefinitions[columnName] ?? {}
this.config.columnDefinitions[columnName] = {
...prev,
type: type
}
await this.saveConfig()
}
async loadConfig() {
while (!this.file) {
await delay(100)
}
this.config = await loadConfig(this.file, this.app.vault)
}
async saveConfig() {
await saveConfig(this.file!, this.config, this.app.vault)
}
private getColumnConfigurations(columns: string[]) {
return columns.map(f => {
if (!this.config) {
return {
field: f
}
}
const def = this.config.columnDefinitions[f]?.type
if (!def || def === 'auto') {
return { field: f }
}
if (def === 'date') {
return {
field: f,
cellDataType: 'dateString'
}
}
return {
field: f,
cellDataType: def
}
})
}
refreshTypes() {
if (this.gridCommunicator) {
const columns = this.result.fields
if (columns && columns.length) {
this.gridCommunicator.gridApi.setGridOption('columnDefs', this.getColumnConfigurations(columns))
}
}
}
moveColumn(name: string, toIndex: number) {
let fields = this.result.fields as Array<string>
fields = fields.filter(f => f !== name)
@ -133,42 +215,82 @@ export class CSVView extends TextFileView {
}
api: any = null;
gridCommunicator: GridRendererCommunicator | null = null
refreshSkip: number = 0
formatWithTypes(d: Record<string, string | boolean | number>) {
Object.entries(this.config.columnDefinitions).forEach(([key, value]) => {
if (!d[key]) {
return
}
if (value.type === 'boolean') {
if (d[key] === 'false' || d[key] === '0') {
d[key] = false
}
d[key] = !!d[key]
}
if (value.type === 'number') {
if (d[key] === null || d[key] === '') {
d[key] = ''
} else {
d[key] = parseFloat(d[key] as string)
}
}
if (value.type === 'date') {
// try parsing
const val = d[key] as string
const date = new Date(val)
d[key] = date.toISOString().split('T')[0]
}
})
return d
}
prepareData() {
const result = parse(this.content, {
header: true,
skipEmptyLines: true,
});
const data = result.data.map((d: any, i) => ({
...this.formatWithTypes(d),
__index: i.toString()
}))
return {
data: data,
fields: result.meta.fields
}
}
loadDataIntoGrid() {
if (this.refreshSkip > Date.now()) {
return
}
requestAnimationFrame(() => {
const result = parse(this.content, {
header: true,
skipEmptyLines: true,
});
const data = result.data.map((d: any, i) => ({
...d,
__index: i.toString()
}))
this.result = {
data: data,
fields: result.meta.fields
}
this.api!.render({
data: data,
columns: result.meta.fields
})
const result = this.prepareData()
this.result = result
this.refreshTypes()
this.api!.render(result)
})
}
isLoading: boolean = false
private async renderCSV() {
if (this.api) {
this.loadDataIntoGrid()
if (this.isLoading) {
if (this.api) {
this.loadDataIntoGrid()
}
return
}
this.isLoading = true
this.contentEl.empty()
const csvEditorDiv = this.contentEl.createDiv({ cls: 'sql-seal-csv-editor' })
const buttonsRow = csvEditorDiv.createDiv({ cls: 'sql-seal-csv-viewer-buttons' })
await this.loadConfig()
if (this.enableEditing) {
const createColumn = buttonsRow.createEl('button', { text: 'Add Column' })
const createRow = buttonsRow.createEl('button', { text: 'Add Row' })
@ -199,16 +321,14 @@ export class CSVView extends TextFileView {
modal.open()
})
const grid = new GridRenderer(this.app, null)
const csvView = this;
const data = this.prepareData()
this.result = data
const api = grid.render({
columnDefs: this.getColumnConfigurations(data.fields ?? []),
defaultColDef: {
editable: this.enableEditing,
valueSetter: (e) => {
e.data[e.column.getUserProvidedColDef()?.headerName!] = e.newValue
return e.newValue
},
headerComponentParams: {
enableMenu: this.enableEditing,
showColumnMenu: function (e: any) {
@ -219,7 +339,7 @@ export class CSVView extends TextFileView {
item.onClick(() => {
const modal = new RenameColumnModal(csvView.app, (res) => {
csvView.renameColumn(
this.column.userProvidedColDef.headerName, res)
this.column.userProvidedColDef.field, res)
})
modal.open()
})
@ -228,7 +348,7 @@ export class CSVView extends TextFileView {
menu.addItem(item => {
item.setTitle('Delete Column')
item.onClick(() => {
const colName = this.column.userProvidedColDef.headerName
const colName = this.column.userProvidedColDef.field
const modal = new DeleteConfirmationModal(csvView.app, `column ${colName}`, () => {
csvView.deleteColumn(colName)
})
@ -236,7 +356,34 @@ export class CSVView extends TextFileView {
})
})
// FIXME: rework it to submenus.
menu.addSeparator()
menu.addItem(item => {
// item.setDisabled(true)
item.setTitle('Data Type')
// item.setIsLabel(true)
const ipfSubmenu = (item as any).setSubmenu();
const types = ['auto', 'text', 'number', 'boolean', 'date'] as ColumnType[]
const colName = this.column.userProvidedColDef.field
const current = csvView.getColumnType(colName)
types.forEach(type => {
ipfSubmenu.addItem((subItem: MenuItem) => {
const checkbox = type === current ? '✓ ' : ''
subItem.setTitle(checkbox + type)
subItem.onClick(() => {
csvView.changeColumnType(colName, type)
csvView.refreshTypes()
csvView.loadDataIntoGrid()
})
})
})
})
const pos = e.getBoundingClientRect();
menu.showAtPosition({ x: pos.x, y: pos.y + 20 })
}
@ -254,7 +401,7 @@ export class CSVView extends TextFileView {
if (!columnName) {
return
}
csvView.moveColumn(columnName?.headerName!, e.toIndex!)
csvView.moveColumn(columnName?.field!, e.toIndex!)
},
domLayout: 'normal',
getRowId: (p) => p.data.__index,
@ -280,9 +427,9 @@ export class CSVView extends TextFileView {
})
menu.showAtMouseEvent(e.event as any)
}
}, gridEl, { cellParser: this.cellParser, sourcePath: this.file?.path || '' })
}, gridEl, { sourcePath: this.file?.path || '' })
this.api = api;
this.loadDataIntoGrid()
this.gridCommunicator = api.communicator
api.render(data)
}
}

View file

@ -295,4 +295,10 @@
.markdown-source-view.mod-cm6.is-readable-line-width .cm-contentContainer.cm-contentContainer>.cm-content>:has(>.block-language-sqlseal table.dataview) {
width: var(--container-dataview-table-width);
max-width: var(--container-table-max-width);
}
/* AG GRID CUSTOMS */
.ag-checkbox-input {
opacity: 1 !important;
}