obsidian-plugin/h-sphere_sql-seal
1
0
Fork 0
mirror of https://github.com/h-sphere/sql-seal.git synced 2026-07-22 10:10:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #13 from h-sphere/feat/signals

Browse source 
Adding Signals
This commit is contained in:
Kacper Kula 2024-11-03 21:04:15 +00:00 committed by GitHub
commit 0223c412ef
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 830 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
jest.config.js
View file

@ -3,5 +3,6 @@ module.exports = {
testEnvironment: "node",
transform: {
"^.+.tsx?$": ["ts-jest",{}],
"^.+.ts?$": ["ts-jest",{}],
},
};

4
main.ts
View file

@ -12,13 +12,13 @@ export default class SqlSealPlugin extends Plugin {
async onload() {
await this.loadSettings();
const sqlSeal = new SqlSeal(this.app, false) // FIXME: set verbose based on the env.
const sqlSeal = new SqlSeal(this.app, true) // FIXME: set verbose based on the env.
this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler())
this.sqlSeal = sqlSeal
// start syncing when files are loaded
this.app.workspace.onLayoutReady(() => {
sqlSeal.db.connect().then(() => {
this.fileSync = new SealFileSync(this.app, sqlSeal, this)
this.fileSync = new SealFileSync(this.app, sqlSeal, this, sqlSeal.tablesManager)
this.fileSync.init()
})
})

14
package-lock.json generated
View file

@ -1,15 +1,14 @@
{
"name": "sqlseal",
"version": "0.6.0",
"version": "0.8.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "sqlseal",
"version": "0.6.0",
"version": "0.8.0",
"license": "MIT",
"dependencies": {
"antlr4": "^4.13.2",
"antlr4ts": "0.5.0-alpha.4",
"better-sqlite3": "^11.2.1",
"json5": "^2.2.3",
@ -3243,15 +3242,6 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/antlr4": {
"version": "4.13.2",
"resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz",
"integrity": "sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=16"
}
},
"node_modules/antlr4ts": {
"version": "0.5.0-alpha.4",
"resolved": "https://registry.npmjs.org/antlr4ts/-/antlr4ts-0.5.0-alpha.4.tgz",

64
src/SealFileSync.ts
View file

@ -1,8 +1,10 @@
import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian";
import { App, EventRef, getAllTags, Plugin, TAbstractFile, TFile } from "obsidian";
import { SqlSeal } from "./sqlSeal";
import { FieldTypes } from "./utils";
import { TablesManager } from "./dataLoader/collections/tablesManager";
import { sanitise } from "./utils/sanitiseColumn";
function fileData(file: TAbstractFile, frontmatter: Record<string, any>) {
function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<string, any>) {
return {
...frontmatter,
path: file.path,
@ -12,17 +14,28 @@ function fileData(file: TAbstractFile, frontmatter: Record<string, any>) {
}
const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
return plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
return Object.fromEntries(
Object.entries(frontmatter)
.map(([v, s]) => ([sanitise(v), s]))
)
}
export class SealFileSync {
private currentSchema: Record<string, FieldTypes> = {}
constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) {
constructor(
public readonly app: App,
private readonly sqlSeal: SqlSeal,
private readonly plugin: Plugin,
private readonly tableManager: TablesManager
) {
plugin.registerEvent(this.app.vault.on('modify', async (file) => {
if (!(file instanceof TFile)) {
return
}
const frontmatter = extractFrontmatterFromFile(file, plugin)
await sleep(100)
const frontmatter = await extractFrontmatterFromFile(file, plugin)
if (this.hasNewColumns(frontmatter)) {
await sleep(1000)
@ -34,13 +47,14 @@ export class SealFileSync {
await this.sqlSeal.db.updateData('files', [fileData(file, frontmatter)])
await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId')
this.sqlSeal.observer.fireObservers('table:files')
this.tableManager.getTableSignal('files')(Date.now())
// Wait 1 second before updating tags table
await sleep(1000)
await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
this.sqlSeal.observer.fireObservers('table:tags')
this.tableManager.getTableSignal('tags')(Date.now())
}))
plugin.registerEvent(this.app.vault.on('create', async (file) => {
@ -60,12 +74,13 @@ export class SealFileSync {
// we need to update the row
await this.sqlSeal.db.insertData('files', [fileData(file, frontmatter)])
this.sqlSeal.observer.fireObservers('table:files')
this.tableManager.getTableSignal('files')(Date.now())
// Wait 1 second before updating tags table
await sleep(1000)
await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
this.sqlSeal.observer.fireObservers('table:tags')
this.tableManager.getTableSignal('tags')(Date.now())
}))
plugin.registerEvent(this.app.vault.on('delete', async (file) => {
@ -74,10 +89,11 @@ export class SealFileSync {
}
await this.sqlSeal.db.deleteData('files', [{ id: file.path }])
this.sqlSeal.observer.fireObservers('table:files')
this.tableManager.getTableSignal('files')(Date.now())
await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId')
this.sqlSeal.observer.fireObservers('table:tags')
this.tableManager.getTableSignal('tags')(Date.now())
}))
plugin.registerEvent(this.app.vault.on('rename', async (file, oldPath) => {
@ -92,12 +108,14 @@ export class SealFileSync {
await this.sqlSeal.db.deleteData('tags', [{ fileId: oldPath }], 'fileId')
await this.sqlSeal.db.insertData('files', [fileData(file, await extractFrontmatterFromFile(file, this.plugin))])
this.sqlSeal.observer.fireObservers('table:files')
this.tableManager.getTableSignal('files')(Date.now())
// Wait 1 second before updating tags table
await sleep(1000)
await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
this.sqlSeal.observer.fireObservers('table:tags')
this.tableManager.getTableSignal('tags')(Date.now())
}))
@ -112,9 +130,18 @@ export class SealFileSync {
}
async getFileTags(file: TFile) {
return (this.app.metadataCache.getFileCache(file)?.tags || [])
.map(t => t.tag)
.map(t => ({ tag: t, fileId: file.path }))
const cache = this.app.metadataCache.getFileCache(file)
if (!cache) {
return []
}
const tags = getAllTags(cache)
if (!tags) {
return []
}
return tags.map((t) => ({
tag: t,
fileId: file.path
}))
}
async init() {
@ -139,8 +166,9 @@ export class SealFileSync {
'fileId': 'TEXT'
})
}
this.sqlSeal.observer.fireObservers('table:files')
this.sqlSeal.observer.fireObservers('table:tags')
this.tableManager.getTableSignal('files')(Date.now())
this.tableManager.getTableSignal('tags')(Date.now())
}
hasNewColumns(newFrontmatter: Record<string, any>) {

77
src/SealObserver.ts
View file

@ -1,77 +0,0 @@
import { Logger } from "./logger";
type Callback = () => void;
export class SealObserver {
private tables: Map<string, Set<Callback>>;
private tags: Map<string, Set<Callback>>;
private logger: Logger
constructor(verbose = false) {
this.tables = new Map();
this.tags = new Map();
this.logger = new Logger(verbose)
}
registerObserver(tableNames: string | string[], observer: Callback, tag?: string) {
if (typeof tableNames === 'string') {
tableNames = [tableNames];
}
tableNames.forEach(tableName => {
if (!this.tables.has(tableName)) {
this.tables.set(tableName, new Set<Callback>());
this.logger.log(`Table "${tableName}" registered.`);
}
const observers = this.tables.get(tableName);
if (observers !== undefined) {
observers.add(observer);
this.logger.log(`Observer registered for "${tableName}".`);
}
if (tag) {
if (!this.tags.has(tag)) {
this.tags.set(tag, new Set<Callback>());
}
const tags = this.tags.get(tag);
if (tags !== undefined) {
tags.add(observer);
}
}
});
}
hasAnyObserver(tableNames: string[]) {
return tableNames.reduce((acc, t) =>
acc || (this.tables.has(t) && this.tables.get(t)!.size > 0),
false)
}
unregisterObserversByTag(tag: string) {
if (this.tags.has(tag)) {
const observers = this.tags.get(tag);
if (observers) {
observers.forEach(observer => {
this.unregisterObserver(observer)
})
this.tags.delete(tag)
}
}
}
unregisterObserver(observer: Callback) {
this.tables.forEach(observers => {
observers.delete(observer);
});
this.logger.log(`Observer unregistered.`);
}
fireObservers(tableName: string) {
if (this.tables.has(tableName)) {
const observers = this.tables.get(tableName);
if (observers) {
observers.forEach(observer => observer());
this.logger.log(`Observers fired for table "${tableName}".`);
}
}
}
}

137
src/SqlSealCodeblockHandler.ts
View file

@ -3,11 +3,12 @@ import { displayData, displayError, displayInfo, displayLoader } from "./ui"
import { resolveFrontmatter } from "./frontmatter"
import { hashString } from "./hash"
import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables"
import { SealObserver } from "./SealObserver"
import { SqlSealDatabase } from "./database"
import { Logger } from "./logger"
import { ParsedLanguage, parseLanguage, TableStatement } from "./grammar/parser"
import { parseLanguage, TableStatement } from "./grammar/parser"
import { SyncModel } from "./models/sync"
import { TablesManager } from "./dataLoader/collections/tablesManager"
import { QueryManager } from "./dataLoader/collections/queryManager"
export class SqlSealCodeblockHandler {
@ -15,102 +16,42 @@ export class SqlSealCodeblockHandler {
return ['files', 'tags']
}
syncModel: SyncModel
constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) {
constructor(
private readonly app: App,
private readonly db: SqlSealDatabase,
private logger: Logger,
private tableManager: TablesManager,
private queryManager: QueryManager
) {
this.syncModel = new SyncModel(db)
}
setupTables(tables: ParsedLanguage['tables'], ctx: MarkdownPostProcessorContext) {
tables.forEach(table => {
this.setupTable(table, ctx)
setupTableSignals(tables: Array<TableStatement>) {
tables.forEach(t => {
this.logger.log(`Registering table ${t.name} -> ${t.url}`)
this.tableManager.registerTable(t.name, t.url)
})
}
setupTable({ url, name }: TableStatement, ctx: MarkdownPostProcessorContext) {
// UPDATING TABLES IF THEY CHANGED SINCE LAST REGISTER
const prefix = hashString(ctx.sourcePath)
const prefixedName = prefixedIfNotGlobal(name, this.globalTables, prefix)
setupQuerySignals({ statement, tables }: ReturnType<typeof updateTables>, el: HTMLElement, ctx: MarkdownPostProcessorContext) {
const frontmatter = resolveFrontmatter(ctx, this.app)
const existing = this.syncModel.getSync(ctx.sourcePath, prefixedName)
this.observer.registerObserver(`file:${url}`, async () => {
// Update table
await this.db.loadDataForDatabaseFromUrl(prefixedName, url, true)
// update when it was updated.
this.syncModel.removeSync(ctx.sourcePath, prefixedName)
this.syncModel.registerSync(ctx.sourcePath, url, prefixedName)
// Fire observers for the table
this.observer.fireObservers(`table:${prefixedName}`)
}, ctx.docId)
if (existing) {
if (existing.url === url) {
// Already synced
// TODO: Check if the file was updated after we synced, if so, we should update it.
requestAnimationFrame(() => {
this.observer.fireObservers(`table:${prefixedName}`)
})
return
} else {
// Old database is no-no. We need to update it.
this.db.db.prepare('DROP TABLE :tablename').run({
tablename: prefixedName
})
this.syncModel.removeSync(ctx.sourcePath, prefixedName)
}
}
this.observer.fireObservers(`file:${url}`)
}
setupSelect(selectStmt: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) {
try {
const prefix = hashString(ctx.sourcePath)
const { statement, tables } = updateTables(selectStmt, this.globalTables, prefix)
const frontmatter = resolveFrontmatter(ctx, this.app)
const renderSelect = async () => {
try {
const stmt = this.db.db.prepare(statement)
const columns = stmt.columns().map(column => column.name);
const data = stmt.all(frontmatter ?? {})
displayData(el, columns, data, this.app)
} catch (e) {
displayError(el, e)
}
}
// Register observer for each table
tables.forEach(table => {
const observer = async () => {
if (!el.isConnected) {
// Unregistering using the context id
this.observer.unregisterObserversByTag(ctx.docId)
}
displayLoader(el)
await renderSelect()
}
this.observer.registerObserver(`table:${table}`, observer, ctx.docId)
})
this.globalTables.forEach(t => this.observer.fireObservers(`table:${t}`))
if (!this.observer.hasAnyObserver(this.globalTables.map(t => `table:${t}`))) {
renderSelect()
}
// Triggering the ones which exist
tables.forEach(t => {
if (this.syncModel.getSync(ctx.sourcePath, t)) {
this.observer.fireObservers(`table:${t}`)
}
})
} catch (e) {
if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) {
displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.')
} else {
const renderSelect = async () => {
try {
const stmt = this.db.db.prepare(statement)
const columns = stmt.columns().map(column => column.name);
const data = stmt.all(frontmatter ?? {})
displayData(el, columns, data, this.app)
} catch (e) {
displayError(el, e)
}
}
const sig = this.queryManager.registerQuery(ctx.docId, tables)
sig(() => {
renderSelect()
})
}
getHandler() {
@ -118,17 +59,25 @@ export class SqlSealCodeblockHandler {
displayLoader(el)
await this.db.connect()
this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers.
// NEW PARSING
try {
const results = parseLanguage(source)
const results = parseLanguage(source)
// FIXME: if not properly parsed, we should display error.
const prefix = hashString(ctx.sourcePath)
this.setupTables(results.tables, ctx)
results.queryPart ? this.setupSelect(results.queryPart, el, ctx) : displayInfo(el, `SQLSeal. ${results.tables.map(t => `${t.name} table definition.`).join(' ')}`)
const prefixedTables = results.tables.map(t => {
return {
...t,
name: prefixedIfNotGlobal(t.name, this.globalTables, prefix)
}
})
this.setupTableSignals(prefixedTables)
if (results.queryPart) {
const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix)
this.setupQuerySignals({ statement, tables }, el, ctx)
}
} catch (e) {
displayError(el, e.toString())
}

53
src/dataLoader/collections/filesManager.ts Normal file
View file

@ -0,0 +1,53 @@
import { Signal, SignalUnsubscriber } from "src/utils/signal";
import { dataTransformer, DataTransformerOut } from "../dataTransformer";
import { csvFileSignal } from "../csvFile";
import { Vault } from "obsidian";
export class FilesManager {
files: Map<string, Signal<DataTransformerOut>> = new Map()
inputFiles: Map<string, Signal<string>> = new Map()
unregisters: Array<SignalUnsubscriber> = []
constructor(private vault: Vault) {
this.vault.on('modify', async (file) => {
if (this.files.has(file.path)) {
this.inputFiles.get(file.path)!(await this.loadFile(file.path))
}
})
}
private async loadFile(url: string) {
const file = this.vault.getFileByPath(url)
if (!file) {
return ''
}
const data = await this.vault.cachedRead(file)
return data
}
destroy() {
this.unregisters.forEach(u => u())
}
getFileSignal(filename: string): Signal<DataTransformerOut> {
if (this.files.has(filename)) {
return this.files.get(filename)!
}
const fileSignal = csvFileSignal(filename)
const signal = dataTransformer(fileSignal)
// FIXME: add here ability to watch changes.
this.files.set(filename, signal)
this.inputFiles.set(filename, fileSignal)
// now firing signal for the file
this.loadFile(filename).then(fileSignal)
return signal
}
addUnregister(unreg: SignalUnsubscriber) {
this.unregisters.push(unreg)
}
}

21
src/dataLoader/collections/queryManager.ts Normal file
View file

@ -0,0 +1,21 @@
import { createSignal, derivedSignal, SignalUnsubscriber, withSignals } from "src/utils/signal";
import { TablesManager } from "./tablesManager";
export class QueryManager {
private registeredQueries: Map<string, SignalUnsubscriber> = new Map()
constructor(private tablesManager: TablesManager) { }
registerQuery(fileId: string, tables: Array<string>) {
if (this.registeredQueries.has(fileId)) {
this.registeredQueries.get(fileId)!()
this.registeredQueries.delete(fileId)
}
const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t, true))
const quertySignal = createSignal<number>()
const unregister = withSignals(...tableSignals)(() => {
quertySignal(Date.now())
})
this.registeredQueries.set(fileId, unregister)
return quertySignal
}
}

48
src/dataLoader/collections/tablesManager.ts Normal file
View file

@ -0,0 +1,48 @@
import { SqlSealDatabase } from "src/database";
import { FilesManager } from "./filesManager";
import { createSignal, Signal, SignalUnsubscriber } from "src/utils/signal";
import { SignalConstants } from "os";
import { linkTableWithFile } from "../tableSignal";
interface Definition {
fileName: string;
unlink: SignalUnsubscriber;
}
export class TablesManager {
private tableLinks: Map<string, Definition> = new Map()
private tableSignals: Map<string, Signal<number>> = new Map()
constructor(private filesManager: FilesManager, private db: SqlSealDatabase) {
}
registerTable(tableName: string, fileName: string) {
if (this.tableLinks.has(tableName)) {
const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)!
if (prevFileName === fileName) {
return
}
unlink()
this.tableLinks.delete(tableName)
}
const fileSignal = this.filesManager.getFileSignal(fileName)
const tableSignal = this.getTableSignal(tableName)
const unlink = linkTableWithFile(fileSignal, tableSignal, tableName, this.db)
this.tableLinks.set(tableName, {
fileName,
unlink
})
}
getTableSignal(tableName: string, failOnUndefined: boolean = false) {
if (!this.tableSignals.has(tableName)) {
if (failOnUndefined) {
throw new Error(`${tableName} does not exist`)
}
this.tableSignals.set(tableName, createSignal<number>())
}
return this.tableSignals.get(tableName)!
}
}

8
src/dataLoader/csvFile.ts Normal file
View file

@ -0,0 +1,8 @@
import { createSignal, Signal } from "src/utils/signal";
export type CSVData = string
export const csvFileSignal = (_file: string): Signal<CSVData> => {
const sig = createSignal<CSVData>()
return sig
}

39
src/dataLoader/dataTransformer.test.ts Normal file
View file

@ -0,0 +1,39 @@
import { describe, it, expect, jest } from '@jest/globals'
import { createSignal } from '../utils/signal'
import { dataTransformer } from './dataTransformer'
describe('Data Transformer', () => {
it('should properly transform basic file', () => {
const s = createSignal<string>()
const dt = dataTransformer(s)
s(`a,b,c
1,2,3
4,5,6`)
expect(dt.value.data).toHaveLength(2)
expect(dt.value.data[0]).toEqual({
a: 1,
b: 2,
c: 3
})
expect(dt.value.types).toEqual({
a: 'INTEGER',
b: 'INTEGER',
c: 'INTEGER'
})
})
it('should properly transform incorrect keys', () => {
const s = createSignal<string>()
const dt = dataTransformer(s)
s(`BEGIN,PLAN,QUERY,RAISE,beak size (mm)
1.5,hello world,343.423,22.34,2`)
expect(dt.value.types).toEqual({
'begin_': 'REAL',
'plan_': 'TEXT',
'query_': 'REAL',
'raise_': 'REAL',
'beak_size__mm_': 'INTEGER'
})
})
})

33
src/dataLoader/dataTransformer.ts Normal file
View file

@ -0,0 +1,33 @@
import { derivedSignal, Signal, SignalEventType } from "../utils/signal"
import { CSVData } from "./csvFile"
import { parse } from 'papaparse'
import { toTypeStatements } from "../utils"
import { sanitise } from "../utils/sanitiseColumn"
export const dataTransformer = (s: Signal<CSVData>) => {
const sig = derivedSignal([s], (csvData) => {
// FIXME: fix header here.
try {
const parsed = parse<Record<string, string>>(csvData, {
header: true,
dynamicTyping: false,
skipEmptyLines: true,
transformHeader: sanitise
})
const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
return typeStatements
} catch (e) {
console.error(e);
return {
data: [],
types: {}
}
}
})
return sig
}
export type DataTransformerOut = SignalEventType<ReturnType<typeof dataTransformer>>

29
src/dataLoader/tableSignal.ts Normal file
View file

@ -0,0 +1,29 @@
import { Signal } from "src/utils/signal";
import { DataTransformerOut } from "./dataTransformer";
import { SqlSealDatabase } from "src/database";
import { FieldTypes } from "src/utils";
export const linkTableWithFile = (dataSig: Signal<DataTransformerOut>, tableSignal: Signal<number>, tableName: string, db: SqlSealDatabase) => {
return dataSig(({ data, types}) => {
// Check if the columns are exactly the same. If not, delete and reinstantiate the table.
// For now always remove and reinstantiate the table
db.dropTable(tableName)
// compute the types of the keys
const columns = Object.entries(types).map(([key, value]) => ({
name: key,
type: value as FieldTypes
}));
// create table again
db.createTableClean(tableName, columns)
// Load all the data
db.insertData(tableName, data)
// Here we do the actual update of the table. If it succeeds, we return it with current date to indicate sync time.
tableSignal(Date.now())
})
}

72
src/database.ts
View file

@ -1,5 +1,5 @@
import Database from "better-sqlite3"
import { App, normalizePath } from "obsidian"
import { App } from "obsidian"
import path from 'path'
import Papa from 'papaparse'
import { prefixedIfNotGlobal } from "./sqlReparseTables"
@ -7,6 +7,12 @@ import { camelCase } from 'lodash'
import { dataToCamelCase, fetchBlobData, FieldTypes, predictJson, predictType, toTypeStatements } from "./utils"
import os from 'os'
import fs from 'fs'
import { sanitise } from "./utils/sanitiseColumn"
export interface FieldDefinition {
name: string;
type: FieldTypes
}
export class SqlSealDatabase {
private savedDatabases: Record<string, any> = {}
@ -172,38 +178,56 @@ export class SqlSealDatabase {
}
async insertData(name: string, inData: Array<Record<string, unknown>>) {
const data = dataToCamelCase(inData)
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
// FIXME: reworking all fields to be camel case
if (!fields || !fields.length) {
return
}
const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`);
const insertMany = this.db.transaction((pData: Array<Record<string, any>>) => {
// FIXME: should we do this as a transaction or not?
pData.forEach(data => {
try {
// update data so all missing fields are set to null
fields.forEach(field => {
if (typeof data[field] === 'boolean') {
data[field] = data[field] ? 1 : 0
} else if (!data[field]) {
data[field] = null
} else if (typeof data[field] === 'object' || Array.isArray(data[field])) {
data[field] = JSON.stringify(data[field])
const columns = Object.keys(data)
const insert = this.db.prepare(`INSERT INTO ${name} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
const d = Object.keys(data).reduce((ret, key) => {
if (typeof data[key] === 'boolean') {
return {
...ret,
[key]: data[key] ? 1 : 0
}
})
insert.run(data)
} catch (e) {
console.error(e, insert, data)
}
}
if (!data[key]) {
return {
...ret,
[key]: null
}
}
if (typeof data[key] === 'object' || Array.isArray(data[key])) {
return {
...ret,
[key]: JSON.stringify(data[key])
}
}
return {
...ret,
[key]: data[key]
}
}, {})
insert.run(d)
})
})
insertMany(data)
insertMany(inData)
}
dropTable(name: string) {
this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
this.savedDatabases[name] = false
}
createTableClean(name: string, fields: Array<FieldDefinition>) {
const sqlFields = fields.map(({ name, type }) => `${name} ${type}`).join(', ')
const createSql = `CREATE TABLE IF NOT EXISTS ${name} (${sqlFields})`
this.db.prepare(createSql).run()
this.savedDatabases[name] = true
}
async createTable(name: string, fields: Record<string, FieldTypes>) {
const transformedFiels = Object.entries(fields).map(([key, type]) => [camelCase(key), type])
const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
const uniqueFields = [...new Map(transformedFiels.map(item =>
[item[0], item])).values()]
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)

22
src/sqlSeal.ts
View file

@ -1,30 +1,28 @@
import { SqlSealDatabase } from "./database";
import { App } from "obsidian";
import { SealObserver } from "./SealObserver";
import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler";
import { Logger } from "./logger";
import { TablesManager } from "./dataLoader/collections/tablesManager";
import { QueryManager } from "./dataLoader/collections/queryManager";
import { FilesManager } from "./dataLoader/collections/filesManager";
export class SqlSeal {
public db: SqlSealDatabase
public observer: SealObserver
public codeBlockHandler: SqlSealCodeblockHandler
public tablesManager: TablesManager
constructor(private readonly app: App, verbose = false) {
this.db = new SqlSealDatabase(app, verbose)
this.observer = new SealObserver(verbose)
this.observeAllFileChanges()
const logger = new Logger(verbose)
this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger)
const fileManager = new FilesManager(this.app.vault)
this.tablesManager = new TablesManager(fileManager, this.db)
const queryManager = new QueryManager(this.tablesManager)
this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, logger, this.tablesManager, queryManager)
// FIXME: handle here changes to files and tags?
}
getHandler() {
return this.codeBlockHandler.getHandler()
}
private observeAllFileChanges() {
// Use fs to observe file changes
this.app.vault.on('modify', async (file) => {
this.observer.fireObservers('file:' + file.path)
})
}
}

55
src/utils/sanitiseColumn.ts Normal file
View file

@ -0,0 +1,55 @@
/**
* Sanitizes a string to be used as a valid SQLite column name.
* Rules implemented:
* 1. Must start with a letter or underscore
* 2. Can only contain letters, numbers, and underscores
* 3. Cannot be a SQLite reserved keyword
* 4. Maximum length is 128 characters (SQLite's practical limit)
* 5. Case-insensitive comparison for reserved keywords
*
* @param input The string to sanitize
* @returns A valid SQLite column name
*/
export function sanitise(input: string): string {
// List of SQLite keywords that cannot be used as column names
const SQLITE_KEYWORDS = new Set([
'ABORT', 'ACTION', 'ADD', 'AFTER', 'ALL', 'ALTER', 'ANALYZE', 'AND', 'AS', 'ASC',
'ATTACH', 'AUTOINCREMENT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BY', 'CASCADE', 'CASE',
'CAST', 'CHECK', 'COLLATE', 'COLUMN', 'COMMIT', 'CONFLICT', 'CONSTRAINT', 'CREATE',
'CROSS', 'CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'DATABASE', 'DEFAULT',
'DEFERRABLE', 'DEFERRED', 'DELETE', 'DESC', 'DETACH', 'DISTINCT', 'DROP', 'EACH',
'ELSE', 'END', 'ESCAPE', 'EXCEPT', 'EXCLUSIVE', 'EXISTS', 'EXPLAIN', 'FAIL',
'FOR', 'FOREIGN', 'FROM', 'FULL', 'GLOB', 'GROUP', 'HAVING', 'IF', 'IGNORE',
'IMMEDIATE', 'IN', 'INDEX', 'INDEXED', 'INITIALLY', 'INNER', 'INSERT', 'INSTEAD',
'INTERSECT', 'INTO', 'IS', 'ISNULL', 'JOIN', 'KEY', 'LEFT', 'LIKE', 'LIMIT',
'MATCH', 'NATURAL', 'NO', 'NOT', 'NOTNULL', 'NULL', 'OF', 'OFFSET', 'ON',
'OR', 'ORDER', 'OUTER', 'PLAN', 'PRAGMA', 'PRIMARY', 'QUERY', 'RAISE',
'RECURSIVE', 'REFERENCES', 'REGEXP', 'REINDEX', 'RELEASE', 'RENAME', 'REPLACE',
'RESTRICT', 'RIGHT', 'ROLLBACK', 'ROW', 'SAVEPOINT', 'SELECT', 'SET',
'TABLE', 'TEMP', 'TEMPORARY', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'UNION',
'UNIQUE', 'UPDATE', 'USING', 'VACUUM', 'VALUES', 'VIEW', 'VIRTUAL', 'WHEN',
'WHERE', 'WITH', 'WITHOUT'
]);
if (!input || typeof input !== 'string') {
return '_empty';
}
// Replace any character that isn't a letter, number, or underscore with underscore
let sanitized = input.replace(/[^a-zA-Z0-9_]/g, '_');
// Ensure it starts with a letter or underscore
if (!/^[a-zA-Z_]/.test(sanitized)) {
sanitized = '_' + sanitized;
}
// Truncate to maximum length
sanitized = sanitized.slice(0, 128);
// If it's a reserved keyword, append underscore
if (SQLITE_KEYWORDS.has(sanitized.toUpperCase())) {
sanitized += '_';
}
return sanitized.toLowerCase();
}

247
src/utils/signal.test.ts Normal file
View file

@ -0,0 +1,247 @@
import { describe, it, expect, jest } from '@jest/globals'
import { createSignal, derivedSignal, withSignals } from './signal';
describe('createSignal', () => {
it('should create a signal with initial value', () => {
const signal = createSignal(10);
expect(signal.value).toBe(10);
});
it('should create a signal without initial value', () => {
const signal = createSignal<number>();
expect(signal.value).toBeUndefined();
});
it('should notify subscribers when value changes', () => {
const signal = createSignal<number>();
const listener = jest.fn();
signal(listener);
signal(42);
expect(listener).toHaveBeenCalledWith(42);
});
it('should call listener immediately with current value if exists', () => {
const signal = createSignal(10);
const listener = jest.fn();
signal(listener);
expect(listener).toHaveBeenCalledWith(10);
});
it('should allow unsubscribing', () => {
const signal = createSignal<number>();
const listener = jest.fn();
const unsubscribe = signal(listener);
signal(42);
expect(listener).toHaveBeenCalledTimes(1);
unsubscribe();
signal(43);
expect(listener).toHaveBeenCalledTimes(1);
});
it('should allow multiple subscribers', () => {
const signal = createSignal<number>();
const listener1 = jest.fn();
const listener2 = jest.fn();
signal(listener1);
signal(listener2);
signal(42);
expect(listener1).toHaveBeenCalledWith(42);
expect(listener2).toHaveBeenCalledWith(42);
});
});
describe('withSignals', () => {
it('should combine multiple signals', () => {
const signal1 = createSignal<number>();
const signal2 = createSignal<string>();
const callback = jest.fn();
withSignals(signal1, signal2)(callback);
signal1(42);
expect(callback).not.toHaveBeenCalled();
signal2('hello');
expect(callback).toHaveBeenCalledWith(42, 'hello');
});
it('should work with initial values', () => {
const signal1 = createSignal(42);
const signal2 = createSignal('hello');
const callback = jest.fn();
withSignals(signal1, signal2)(callback);
expect(callback).toHaveBeenCalledWith(42, 'hello');
});
it('should update when any signal changes', () => {
const signal1 = createSignal(42);
const signal2 = createSignal('hello');
const callback = jest.fn();
withSignals(signal1, signal2)(callback);
signal1(43);
expect(callback).toHaveBeenCalledWith(43, 'hello');
});
it('should allow unsubscribing from all signals', () => {
const signal1 = createSignal<number>();
const signal2 = createSignal<string>();
const callback = jest.fn();
const unsubscribe = withSignals(signal1, signal2)(callback);
signal1(42);
signal2('hello');
expect(callback).toHaveBeenCalledTimes(1);
unsubscribe();
signal1(43);
signal2('world');
expect(callback).toHaveBeenCalledTimes(1);
});
});
describe('derivedSignal', () => {
it('should compute derived value', () => {
const count = createSignal(5);
const multiplier = createSignal(2);
const product = derivedSignal(
[count, multiplier],
(c, m) => c * m
);
expect(product.value).toBe(10);
});
it('should update when source signals change', () => {
const count = createSignal(5);
const multiplier = createSignal(2);
const product = derivedSignal(
[count, multiplier],
(c, m) => c * m
);
const listener = jest.fn();
product(listener);
count(10);
expect(product.value).toBe(20);
expect(listener).toHaveBeenCalledWith(20);
multiplier(3);
expect(product.value).toBe(30);
expect(listener).toHaveBeenCalledWith(30);
});
it('should handle undefined initial values', () => {
const signal1 = createSignal<number>();
const signal2 = createSignal<number>();
const sum = derivedSignal(
[signal1, signal2],
(a, b) => a + b
);
expect(sum.value).toBeUndefined();
signal1(5);
expect(sum.value).toBeUndefined();
signal2(3);
expect(sum.value).toBe(8);
});
it('should work with complex derivations', () => {
const firstName = createSignal('John');
const lastName = createSignal('Doe');
const age = createSignal(25);
const person = derivedSignal(
[firstName, lastName, age],
(first, last, a) => ({
fullName: `${first} ${last}`,
age: a,
isAdult: a >= 18
})
);
expect(person.value).toEqual({
fullName: 'John Doe',
age: 25,
isAdult: true
});
firstName('Jane');
expect(person.value).toEqual({
fullName: 'Jane Doe',
age: 25,
isAdult: true
});
});
it('should chain derived signals', () => {
const base = createSignal(5);
const doubled = derivedSignal([base], n => n * 2);
const final = derivedSignal([doubled], n => n + 10);
expect(final.value).toBe(20);
base(10);
expect(doubled.value).toBe(20);
expect(final.value).toBe(30);
});
it('should support array transformations', () => {
const items = createSignal(['apple', 'banana', 'orange']);
const filter = createSignal('an');
const filtered = derivedSignal(
[items, filter],
(list, f) => list.filter(item =>
item.toLowerCase().includes(f.toLowerCase())
)
);
expect(filtered.value).toEqual(['banana', 'orange']);
filter('ap');
expect(filtered.value).toEqual(['apple']);
items(['grape', 'apple', 'mango']);
expect(filtered.value).toEqual(['grape', 'apple']);
});
});
describe('type safety', () => {
it('should maintain proper types in derived signals', () => {
const numberSignal = createSignal<number>(42);
const stringSignal = createSignal<string>('hello');
// This should type check
const derived = derivedSignal(
[numberSignal, stringSignal],
(num, str) => ({
number: num, // Should be typed as number
string: str, // Should be typed as string
combined: `${str}${num}`
})
);
expect(derived.value).toEqual({
number: 42,
string: 'hello',
combined: 'hello42'
});
});
});

145
src/utils/signal.ts Normal file
View file

@ -0,0 +1,145 @@
/**
* Enhanced signal types to include value access
*/
export interface SignalListener<T> {
(event: T): void;
}
export interface SignalUnsubscriber {
(): void;
}
export interface SignalSubscriber<T> {
(listener: SignalListener<T>): SignalUnsubscriber;
}
export interface SignalDispatcher<T> {
(event: T): void;
}
export interface SignalValue<T> {
readonly value: T;
}
export type Signal<T> = SignalSubscriber<T> & SignalDispatcher<T> & SignalValue<T>;
export type SignalReturn = SignalUnsubscriber & void;
export function createSignal<T>(): Signal<T>;
export function createSignal<T>(initialValue: T): Signal<T>;
export function createSignal<T>(initialValue?: T): Signal<T> {
const subscribers = new Set<SignalListener<T>>();
let currentValue: T | undefined = initialValue;
const signal = ((eventOrListener: T | SignalListener<T>): any => {
if (typeof eventOrListener === 'function') {
subscribers.add(eventOrListener as SignalListener<T>);
// Call the listener immediately with current value if it exists
if (currentValue !== undefined) {
(eventOrListener as SignalListener<T>)(currentValue);
}
return () => { subscribers.delete(eventOrListener as SignalListener<T>) };
} else {
currentValue = eventOrListener;
subscribers.forEach(listener => listener(eventOrListener));
}
}) as Signal<T>;
// Add value getter
Object.defineProperty(signal, 'value', {
get: () => currentValue,
enumerable: true
});
return signal;
}
/**
* Utility types for handling async compute functions
*/
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T;
type ComputeReturnType<Args extends any[], R> =
| ((...args: Args) => R)
| ((...args: Args) => Promise<R>);
/**
* Utility type to extract the event type from a Signal
*/
export type SignalEventType<T> = T extends Signal<infer E> ? E : never;
/**
* Creates a new signal that derives its value from other signals
* Supports both synchronous and asynchronous compute functions
*/
export function derivedSignal<Signals extends Signal<any>[], R>(
signals: [...Signals],
compute: ComputeReturnType<{ [K in keyof Signals]: SignalEventType<Signals[K]> }, R>
): Signal<UnwrapPromise<R>> {
const derivedSignal = createSignal<UnwrapPromise<R>>();
let computeVersion = 0;
let isComputing = false;
const updateValue = async (version: number, values: { [K in keyof Signals]: SignalEventType<Signals[K]> }) => {
if (isComputing) return;
isComputing = true;
try {
const result = compute(...Object.values(values) as { [K in keyof Signals]: SignalEventType<Signals[K]> });
const computedValue = result instanceof Promise ? await result : result;
// Only update if this is still the latest computation
if (version === computeVersion) {
derivedSignal(computedValue as UnwrapPromise<R>);
}
} catch (error) {
console.error('Error in derived signal computation:', error);
} finally {
isComputing = false;
}
};
// Compute initial value if all source signals have values
const initialValues = signals.map(s => s.value);
if (!initialValues.includes(undefined)) {
updateValue(computeVersion, initialValues as { [K in keyof Signals]: SignalEventType<Signals[K]> });
}
// Handle updates
withSignals(...signals)((...args) => {
computeVersion++;
updateValue(computeVersion, args as { [K in keyof Signals]: SignalEventType<Signals[K]> });
});
return derivedSignal;
}
/**
* Takes multiple signals and returns a function that accepts a callback
* which will receive the values from those signals
*/
export function withSignals<Signals extends Signal<any>[]>(
...signals: Signals
): <R>(
callback: (...values: { [K in keyof Signals]: SignalEventType<Signals[K]> }) => R
) => SignalUnsubscriber {
return (callback) => {
const unsubscribers: SignalUnsubscriber[] = [];
const values = new Array(signals.length) as { [K in keyof Signals]: SignalEventType<Signals[K]> };
let initialized = new Array(signals.length).fill(false);
signals.forEach((signal, index) => {
const unsubscribe = signal((value) => {
values[index] = value as { [K in keyof Signals]: SignalEventType<Signals[K]> }[number];
initialized[index] = true;
if (initialized.every(Boolean)) {
callback(...(values as unknown as { [K in keyof Signals]: SignalEventType<Signals[K]> }));
}
});
unsubscribers.push(unsubscribe);
});
return () => {
unsubscribers.forEach(unsubscribe => unsubscribe());
};
};
}

5
yarn.lock
View file

@ -1428,11 +1428,6 @@ ansi-styles@^5.0.0:
resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz"
integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
antlr4@^4.13.2:
version "4.13.2"
resolved "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz"
integrity sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==
antlr4ts-cli@0.5.0-alpha.4:
version "0.5.0-alpha.4"
resolved "https://registry.npmjs.org/antlr4ts-cli/-/antlr4ts-cli-0.5.0-alpha.4.tgz"