feat: reworking communication to signals and fixing issues with some keys

jest.config.js

@@ -3,5 +3,6 @@ module.exports = {
   testEnvironment: "node",
   transform: {
     "^.+.tsx?$": ["ts-jest",{}],
+    "^.+.ts?$": ["ts-jest",{}],
   },
 };

main.ts

@@ -18,7 +18,7 @@ export default class SqlSealPlugin extends Plugin {
 		// start syncing when files are loaded
 		this.app.workspace.onLayoutReady(() => {
 			sqlSeal.db.connect().then(() => {
-				this.fileSync = new SealFileSync(this.app, sqlSeal, this)
+				this.fileSync = new SealFileSync(this.app, sqlSeal, this, sqlSeal.tablesManager)
 			this.fileSync.init()
 			})
 		})

package-lock.json

@@ -1,15 +1,14 @@
 {
   "name": "sqlseal",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "sqlseal",
-      "version": "0.6.0",
+      "version": "0.8.0",
       "license": "MIT",
       "dependencies": {
-        "antlr4": "^4.13.2",
         "antlr4ts": "0.5.0-alpha.4",
         "better-sqlite3": "^11.2.1",
         "json5": "^2.2.3",
@@ -3243,15 +3242,6 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/antlr4": {
-      "version": "4.13.2",
-      "resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz",
-      "integrity": "sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==",
-      "license": "BSD-3-Clause",
-      "engines": {
-        "node": ">=16"
-      }
-    },
     "node_modules/antlr4ts": {
       "version": "0.5.0-alpha.4",
       "resolved": "https://registry.npmjs.org/antlr4ts/-/antlr4ts-0.5.0-alpha.4.tgz",

src/SealFileSync.ts

@@ -1,6 +1,8 @@
 import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian";
 import { SqlSeal } from "./sqlSeal";
 import { FieldTypes } from "./utils";
+import { TablesManager } from "./dataLoader/collections/tablesManager";
+import { sanitise } from "./utils/sanitiseColumn";
 
 function fileData(file: TAbstractFile, frontmatter: Record<string, any>) {
     return {
@@ -12,17 +14,28 @@ function fileData(file: TAbstractFile, frontmatter: Record<string, any>) {
 }
 
 const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
-    return plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
+    const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
+
+    return Object.fromEntries(
+        Object.entries(frontmatter)
+            .map(([v, s]) => ([sanitise(v), s]))
+    )
 }
 
 export class SealFileSync {
     private currentSchema: Record<string, FieldTypes> = {}
-    constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) {
+    constructor(
+        public readonly app: App,
+        private readonly sqlSeal: SqlSeal,
+        private readonly plugin: Plugin,
+        private readonly tableManager: TablesManager
+    ) {
         plugin.registerEvent(this.app.vault.on('modify', async (file) => {
             if (!(file instanceof TFile)) {
                 return
             }
-            const frontmatter = extractFrontmatterFromFile(file, plugin)
+            await sleep(100)
+            const frontmatter = await extractFrontmatterFromFile(file, plugin)
 
             if (this.hasNewColumns(frontmatter)) {
                 await sleep(1000)
@@ -35,12 +48,15 @@ export class SealFileSync {
             await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId')
 
             this.sqlSeal.observer.fireObservers('table:files')
+            this.tableManager.getTableSignal('files')(Date.now())
 
 
             // Wait 1 second before updating tags table
             await sleep(1000)
             await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
             this.sqlSeal.observer.fireObservers('table:tags')
+            this.tableManager.getTableSignal('tags')(Date.now())
+
         }))
 
         plugin.registerEvent(this.app.vault.on('create', async (file) => {
@@ -61,11 +77,14 @@ export class SealFileSync {
             // we need to update the row
             await this.sqlSeal.db.insertData('files', [fileData(file, frontmatter)])
             this.sqlSeal.observer.fireObservers('table:files')
+            this.tableManager.getTableSignal('files')(Date.now())
 
             // Wait 1 second before updating tags table
             await sleep(1000)
             await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
             this.sqlSeal.observer.fireObservers('table:tags')
+            this.tableManager.getTableSignal('tags')(Date.now())
+
         }))
         
         plugin.registerEvent(this.app.vault.on('delete', async (file) => {
@@ -75,9 +94,12 @@ export class SealFileSync {
 
             await this.sqlSeal.db.deleteData('files', [{ id: file.path }])
             this.sqlSeal.observer.fireObservers('table:files')
+            this.tableManager.getTableSignal('files')(Date.now())
+
 
             await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId')
             this.sqlSeal.observer.fireObservers('table:tags')
+            this.tableManager.getTableSignal('tags')(Date.now())
         }))
 
         plugin.registerEvent(this.app.vault.on('rename', async (file, oldPath) => {
@@ -93,11 +115,15 @@ export class SealFileSync {
 
             await this.sqlSeal.db.insertData('files', [fileData(file, await extractFrontmatterFromFile(file, this.plugin))])
             this.sqlSeal.observer.fireObservers('table:files')
+            this.tableManager.getTableSignal('files')(Date.now())
+
 
             // Wait 1 second before updating tags table
             await sleep(1000)
             await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
             this.sqlSeal.observer.fireObservers('table:tags')
+            this.tableManager.getTableSignal('tags')(Date.now())
+
 
         }))
 
@@ -141,6 +167,9 @@ export class SealFileSync {
         }
         this.sqlSeal.observer.fireObservers('table:files')
         this.sqlSeal.observer.fireObservers('table:tags')
+        this.tableManager.getTableSignal('files')(Date.now())
+        this.tableManager.getTableSignal('tags')(Date.now())
+
     }
 
     hasNewColumns(newFrontmatter: Record<string, any>) {

src/SqlSealCodeblockHandler.ts

@@ -8,6 +8,8 @@ import { SqlSealDatabase } from "./database"
 import { Logger } from "./logger"
 import { ParsedLanguage, parseLanguage, TableStatement } from "./grammar/parser"
 import { SyncModel } from "./models/sync"
+import { TablesManager } from "./dataLoader/collections/tablesManager"
+import { QueryManager } from "./dataLoader/collections/queryManager"
 
 
 export class SqlSealCodeblockHandler {
@@ -15,7 +17,14 @@ export class SqlSealCodeblockHandler {
         return ['files', 'tags']
     }
     syncModel: SyncModel
-    constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) {
+    constructor(
+        private readonly app: App,
+        private readonly db: SqlSealDatabase,
+        private readonly observer: SealObserver,
+        private logger: Logger,
+        private tableManager: TablesManager,
+        private queryManager: QueryManager
+    ) {
         this.syncModel = new SyncModel(db)
     }
 
@@ -66,7 +75,7 @@ export class SqlSealCodeblockHandler {
 
     setupSelect(selectStmt: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) {
         try {
-        const prefix = hashString(ctx.sourcePath)
+            const prefix = hashString(ctx.sourcePath)
             const { statement, tables } = updateTables(selectStmt, this.globalTables, prefix)
             const frontmatter = resolveFrontmatter(ctx, this.app)
 
@@ -113,22 +122,59 @@ export class SqlSealCodeblockHandler {
         }
     }
 
+    setupTableSignals(tables: Array<TableStatement>) {
+        tables.forEach(t => {
+            this.logger.log(`Registering table ${t.name} -> ${t.url}`)
+            this.tableManager.registerTable(t.name, t.url)
+        })
+    }
+
+    setupQuerySignals({ statement, tables }: ReturnType<typeof updateTables>, el: HTMLElement, ctx: MarkdownPostProcessorContext) {
+        const frontmatter = resolveFrontmatter(ctx, this.app)
+
+        const renderSelect = async () => {
+            try {
+                const stmt = this.db.db.prepare(statement)
+                const columns = stmt.columns().map(column => column.name);
+                const data = stmt.all(frontmatter ?? {})
+                displayData(el, columns, data, this.app)
+            } catch (e) {
+                displayError(el, e)
+            }
+        }
+
+
+        const sig = this.queryManager.registerQuery(ctx.docId, tables)
+        sig(() => {
+            renderSelect()
+        })
+    }
+
     getHandler() {
         return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
 
             displayLoader(el)
             await this.db.connect()
-            this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers.
+            // this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers.
 
-            // NEW PARSING
             try {
-            const results = parseLanguage(source)
+                const results = parseLanguage(source)
 
-            // FIXME: if not properly parsed, we should display error.
+                const prefix = hashString(ctx.sourcePath)
 
-            this.setupTables(results.tables, ctx)
-            results.queryPart ? this.setupSelect(results.queryPart, el, ctx) : displayInfo(el, `SQLSeal. ${results.tables.map(t => `${t.name} table definition.`).join(' ')}`)
+                const prefixedTables = results.tables.map(t => {
+                    return {
+                        ...t,
+                        name: prefixedIfNotGlobal(t.name, this.globalTables, prefix)
+                    }
+                })
 
+                this.setupTableSignals(prefixedTables)
+
+                if (results.queryPart) {
+                    const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix)
+                    this.setupQuerySignals({ statement, tables }, el, ctx)
+                }
             } catch (e) {
                 displayError(el, e.toString())
             }

src/dataLoader/collections/filesManager.ts

@@ -0,0 +1,54 @@
+import { Signal, SignalUnsubscriber } from "src/utils/signal";
+import { dataTransformer, DataTransformerOut } from "../dataTransformer";
+import { csvFileSignal } from "../csvFile";
+import { Vault } from "obsidian";
+
+export class FilesManager {
+    files: Map<string, Signal<DataTransformerOut>> = new Map()
+    inputFiles: Map<string, Signal<string>> = new Map()
+    unregisters: Array<SignalUnsubscriber> = []
+    constructor(private vault: Vault) {
+        this.vault.on('modify', async (file) => {
+            if (this.files.has(file.path)) {
+                console.log(`Modified ${file.path}`)
+                this.inputFiles.get(file.path)!(await this.loadFile(file.path))
+            }
+        })
+    }
+
+    private async loadFile(url: string) {
+        const file = this.vault.getFileByPath(url)
+
+        if (!file) {
+            return ''
+        }
+        const data = await this.vault.cachedRead(file)
+        return data
+    }
+
+    destroy() {
+        this.unregisters.forEach(u => u())
+    }
+
+    getFileSignal(filename: string): Signal<DataTransformerOut> {
+        if (this.files.has(filename)) {
+            return this.files.get(filename)!
+        }
+        const fileSignal = csvFileSignal(filename)
+        const signal = dataTransformer(fileSignal)
+
+        // FIXME: add here ability to watch changes.
+
+        this.files.set(filename, signal)
+        this.inputFiles.set(filename, fileSignal)
+
+        // now firing signal for the file
+        this.loadFile(filename).then(fileSignal)
+
+        return signal
+    }
+
+    addUnregister(unreg: SignalUnsubscriber) {
+        this.unregisters.push(unreg)
+    }
+}

src/dataLoader/collections/queryManager.ts

@@ -0,0 +1,21 @@
+import { createSignal, derivedSignal, SignalUnsubscriber, withSignals } from "src/utils/signal";
+import { TablesManager } from "./tablesManager";
+
+export class QueryManager {
+    private registeredQueries: Map<string, SignalUnsubscriber> = new Map()
+    constructor(private tablesManager: TablesManager) { }
+
+    registerQuery(fileId: string, tables: Array<string>) {
+        if (this.registeredQueries.has(fileId)) {
+            this.registeredQueries.get(fileId)!()
+            this.registeredQueries.delete(fileId)
+        }
+        const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t))
+        const quertySignal = createSignal<number>()
+        const unregister = withSignals(...tableSignals)(() => {
+            quertySignal(Date.now())
+        })
+        this.registeredQueries.set(fileId, unregister)
+        return quertySignal
+    }
+}

src/dataLoader/collections/tablesManager.ts

@@ -0,0 +1,47 @@
+import { SqlSealDatabase } from "src/database";
+import { FilesManager } from "./filesManager";
+import { createSignal, Signal, SignalUnsubscriber } from "src/utils/signal";
+import { SignalConstants } from "os";
+import { linkTableWithFile } from "../tableSignal";
+
+interface Definition {
+    fileName: string;
+    unlink: SignalUnsubscriber;
+}
+
+
+export class TablesManager {
+    private tableLinks: Map<string, Definition> = new Map()
+    private tableSignals: Map<string, Signal<number>> = new Map()
+    constructor(private filesManager: FilesManager, private db: SqlSealDatabase) {
+
+    }
+
+    registerTable(tableName: string, fileName: string) {
+        if (this.tableLinks.has(tableName)) {
+            const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)!
+            if (prevFileName === fileName) {
+                console.log('Same filename', fileName)
+                return
+            }
+            console.log('Different Filename')
+            unlink()
+            this.tableLinks.delete(tableName)
+        }
+        const fileSignal = this.filesManager.getFileSignal(fileName)
+        const tableSignal = this.getTableSignal(tableName)
+
+        const unlink = linkTableWithFile(fileSignal, tableSignal, tableName, this.db)
+        this.tableLinks.set(tableName, {
+            fileName,
+            unlink
+        })
+    }
+
+    getTableSignal(tableName: string) {
+        if (!this.tableSignals.has(tableName)) {
+            this.tableSignals.set(tableName, createSignal<number>())
+        }
+        return this.tableSignals.get(tableName)!
+    }
+}

src/dataLoader/csvFile.ts

@@ -0,0 +1,8 @@
+import { createSignal, Signal } from "src/utils/signal";
+
+export type CSVData = string
+
+export const csvFileSignal = (_file: string): Signal<CSVData> => {
+    const sig = createSignal<CSVData>()
+    return sig
+}

src/dataLoader/dataTransformer.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect, jest } from '@jest/globals'
+import { createSignal } from '../utils/signal'
+import { dataTransformer } from './dataTransformer'
+
+describe('Data Transformer', () => {
+    it('should properly transform basic file', () => {
+        const s = createSignal<string>()
+        const dt = dataTransformer(s)
+        s(`a,b,c
+1,2,3
+4,5,6`)
+        console.log(dt.value)
+        expect(dt.value.data).toHaveLength(2)
+        expect(dt.value.data[0]).toEqual({
+            a: 1,
+            b: 2,
+            c: 3
+        })
+
+        expect(dt.value.types).toEqual({
+            a: 'INTEGER',
+            b: 'INTEGER',
+            c: 'INTEGER'
+        })
+    })
+
+    it('should properly transform incorrect keys', () => {
+        const s = createSignal<string>()
+        const dt = dataTransformer(s)
+        s(`BEGIN,PLAN,QUERY,RAISE,beak size (mm)
+1.5,hello world,343.423,22.34,2`)
+        expect(dt.value.types).toEqual({
+            'BEGIN_': 'REAL',
+            'PLAN_': 'TEXT',
+            'QUERY_': 'REAL',
+            'RAISE_': 'REAL',
+            'beak_size__mm_': 'INTEGER'
+        })
+    })
+})

src/dataLoader/dataTransformer.ts

@@ -0,0 +1,33 @@
+import { derivedSignal, Signal, SignalEventType } from "../utils/signal"
+import { CSVData } from "./csvFile"
+import { parse } from 'papaparse'
+import { toTypeStatements } from "../utils"
+import { sanitise } from "../utils/sanitiseColumn"
+
+
+export const dataTransformer = (s: Signal<CSVData>) => {
+    const sig =  derivedSignal([s], (csvData) => {
+        // FIXME: fix header here.
+        try {
+            const parsed = parse<Record<string, string>>(csvData, {
+                header: true,
+                dynamicTyping: false,
+                skipEmptyLines: true,
+                transformHeader: sanitise
+            })
+
+            const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
+            return typeStatements
+        } catch (e) {
+            console.error(e);
+            return {
+                data: [],
+                types: {}
+            }
+        }
+    })
+
+    return sig
+}
+
+export type DataTransformerOut = SignalEventType<ReturnType<typeof dataTransformer>>

src/dataLoader/tableSignal.ts

@@ -0,0 +1,29 @@
+import { Signal } from "src/utils/signal";
+import { DataTransformerOut } from "./dataTransformer";
+import { SqlSealDatabase } from "src/database";
+import { FieldTypes } from "src/utils";
+
+export const linkTableWithFile = (dataSig: Signal<DataTransformerOut>, tableSignal: Signal<number>, tableName: string, db: SqlSealDatabase) => {
+    return dataSig(({ data, types}) => {
+
+        // Check if the columns are exactly the same. If not, delete and reinstantiate the table.
+
+        // For now always remove and reinstantiate the table
+        db.dropTable(tableName)
+
+        // compute the types of the keys
+        const columns = Object.entries(types).map(([key, value]) => ({
+            name: key,
+            type: value as FieldTypes
+          }));
+
+        // create table again
+        db.createTableClean(tableName, columns)
+
+        // Load all the data
+        db.insertData(tableName, data)
+
+        // Here we do the actual update of the table. If it succeeds, we return it with current date to indicate sync time.
+        tableSignal(Date.now())
+    })
+}

src/database.ts

@@ -1,5 +1,5 @@
 import Database from "better-sqlite3"
-import { App, normalizePath } from "obsidian"
+import { App } from "obsidian"
 import path from 'path'
 import Papa from 'papaparse'
 import { prefixedIfNotGlobal } from "./sqlReparseTables"
@@ -7,6 +7,12 @@ import { camelCase } from 'lodash'
 import { dataToCamelCase, fetchBlobData, FieldTypes, predictJson, predictType, toTypeStatements } from "./utils"
 import os from 'os'
 import fs from 'fs'
+import { sanitise } from "./utils/sanitiseColumn"
+
+export interface FieldDefinition {
+    name: string;
+    type: FieldTypes
+}
 
 export class SqlSealDatabase {
     private savedDatabases: Record<string, any> = {}
@@ -172,38 +178,43 @@ export class SqlSealDatabase {
     }
 
     async insertData(name: string, inData: Array<Record<string, unknown>>) {
-        const data = dataToCamelCase(inData)
-        const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
-        // FIXME: reworking all fields to be camel case
-        if (!fields || !fields.length) {
-            return
-        }
-        const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`);
         const insertMany = this.db.transaction((pData: Array<Record<string, any>>) => {
+            // FIXME: should we do this as a transaction or not?
             pData.forEach(data => {
-                try {
-                    // update data so all missing fields are set to null
-                    fields.forEach(field => {
-                        if (typeof data[field] === 'boolean') {
-                            data[field] = data[field] ? 1 : 0
-                        } else if (!data[field]) {
-                            data[field] = null
-                        } else  if (typeof data[field] === 'object' || Array.isArray(data[field])) {
-                            data[field] = JSON.stringify(data[field])
-                        }
-                    })
-                    insert.run(data)
-                } catch (e) {
-                    console.error(e, insert, data)
-                }
+                const columns = Object.keys(data)
+                const insert = this.db.prepare(`INSERT INTO ${name} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
+                const d = Object.keys(data).reduce((ret, key) => {
+                    if (typeof data[key] === 'boolean') {
+                        return data[key] ? 1 : 0;
+                    }
+                    if (!data[key]) {
+                        return null
+                    }
+                    if (typeof data[key] === 'object' || Array.isArray(data[key])) {
+                        return JSON.stringify(data[key])
+                    }
+                }, {})
+                insert.run(data)
             })
         })
 
-        insertMany(data)
+        insertMany(inData)
+    }
+
+    dropTable(name: string) {
+        this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
+        this.savedDatabases[name] = false
+    }
+
+    createTableClean(name: string, fields: Array<FieldDefinition>) {
+        const sqlFields = fields.map(({ name, type }) => `${name} ${type}`).join(', ')
+        const createSql = `CREATE TABLE IF NOT EXISTS ${name} (${sqlFields})`
+        this.db.prepare(createSql).run()
+        this.savedDatabases[name] = true
     }
 
     async createTable(name: string, fields: Record<string, FieldTypes>) {
-        const transformedFiels = Object.entries(fields).map(([key, type]) => [camelCase(key), type])
+        const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
         const uniqueFields = [...new Map(transformedFiels.map(item =>
             [item[0], item])).values()]
         const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)

src/sqlSeal.ts

@@ -3,17 +3,26 @@ import { App } from "obsidian";
 import { SealObserver } from "./SealObserver";
 import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler";
 import { Logger } from "./logger";
+import { TablesManager } from "./dataLoader/collections/tablesManager";
+import { QueryManager } from "./dataLoader/collections/queryManager";
+import { FilesManager } from "./dataLoader/collections/filesManager";
 
 export class SqlSeal {
     public db: SqlSealDatabase
     public observer: SealObserver
     public codeBlockHandler: SqlSealCodeblockHandler
+    public tablesManager: TablesManager
     constructor(private readonly app: App, verbose = false) {
         this.db = new SqlSealDatabase(app, verbose)
         this.observer = new SealObserver(verbose)
         this.observeAllFileChanges()
         const logger = new Logger(verbose)
-        this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger)
+
+        const fileManager = new FilesManager(this.app.vault)
+        this.tablesManager = new TablesManager(fileManager, this.db)
+        const queryManager = new QueryManager(this.tablesManager)
+
+        this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger, this.tablesManager, queryManager)
         // FIXME: handle here changes to files and tags?
     }
 

src/utils/sanitiseColumn.ts

@@ -0,0 +1,55 @@
+/**
+ * Sanitizes a string to be used as a valid SQLite column name.
+ * Rules implemented:
+ * 1. Must start with a letter or underscore
+ * 2. Can only contain letters, numbers, and underscores
+ * 3. Cannot be a SQLite reserved keyword
+ * 4. Maximum length is 128 characters (SQLite's practical limit)
+ * 5. Case-insensitive comparison for reserved keywords
+ * 
+ * @param input The string to sanitize
+ * @returns A valid SQLite column name
+ */
+export function sanitise(input: string): string {
+    // List of SQLite keywords that cannot be used as column names
+    const SQLITE_KEYWORDS = new Set([
+        'ABORT', 'ACTION', 'ADD', 'AFTER', 'ALL', 'ALTER', 'ANALYZE', 'AND', 'AS', 'ASC',
+        'ATTACH', 'AUTOINCREMENT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BY', 'CASCADE', 'CASE',
+        'CAST', 'CHECK', 'COLLATE', 'COLUMN', 'COMMIT', 'CONFLICT', 'CONSTRAINT', 'CREATE',
+        'CROSS', 'CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'DATABASE', 'DEFAULT',
+        'DEFERRABLE', 'DEFERRED', 'DELETE', 'DESC', 'DETACH', 'DISTINCT', 'DROP', 'EACH',
+        'ELSE', 'END', 'ESCAPE', 'EXCEPT', 'EXCLUSIVE', 'EXISTS', 'EXPLAIN', 'FAIL',
+        'FOR', 'FOREIGN', 'FROM', 'FULL', 'GLOB', 'GROUP', 'HAVING', 'IF', 'IGNORE',
+        'IMMEDIATE', 'IN', 'INDEX', 'INDEXED', 'INITIALLY', 'INNER', 'INSERT', 'INSTEAD',
+        'INTERSECT', 'INTO', 'IS', 'ISNULL', 'JOIN', 'KEY', 'LEFT', 'LIKE', 'LIMIT',
+        'MATCH', 'NATURAL', 'NO', 'NOT', 'NOTNULL', 'NULL', 'OF', 'OFFSET', 'ON',
+        'OR', 'ORDER', 'OUTER', 'PLAN', 'PRAGMA', 'PRIMARY', 'QUERY', 'RAISE',
+        'RECURSIVE', 'REFERENCES', 'REGEXP', 'REINDEX', 'RELEASE', 'RENAME', 'REPLACE',
+        'RESTRICT', 'RIGHT', 'ROLLBACK', 'ROW', 'SAVEPOINT', 'SELECT', 'SET',
+        'TABLE', 'TEMP', 'TEMPORARY', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'UNION',
+        'UNIQUE', 'UPDATE', 'USING', 'VACUUM', 'VALUES', 'VIEW', 'VIRTUAL', 'WHEN',
+        'WHERE', 'WITH', 'WITHOUT'
+    ]);
+
+    if (!input || typeof input !== 'string') {
+        return '_empty';
+    }
+
+    // Replace any character that isn't a letter, number, or underscore with underscore
+    let sanitized = input.replace(/[^a-zA-Z0-9_]/g, '_');
+
+    // Ensure it starts with a letter or underscore
+    if (!/^[a-zA-Z_]/.test(sanitized)) {
+        sanitized = '_' + sanitized;
+    }
+
+    // Truncate to maximum length
+    sanitized = sanitized.slice(0, 128);
+
+    // If it's a reserved keyword, append underscore
+    if (SQLITE_KEYWORDS.has(sanitized.toUpperCase())) {
+        sanitized += '_';
+    }
+
+    return sanitized;
+}

src/utils/signal.test.ts

@@ -0,0 +1,247 @@
+import { describe, it, expect, jest } from '@jest/globals'
+import { createSignal, derivedSignal, withSignals } from './signal';
+
+describe('createSignal', () => {
+  it('should create a signal with initial value', () => {
+    const signal = createSignal(10);
+    expect(signal.value).toBe(10);
+  });
+
+  it('should create a signal without initial value', () => {
+    const signal = createSignal<number>();
+    expect(signal.value).toBeUndefined();
+  });
+
+  it('should notify subscribers when value changes', () => {
+    const signal = createSignal<number>();
+    const listener = jest.fn();
+
+    signal(listener);
+    signal(42);
+
+    expect(listener).toHaveBeenCalledWith(42);
+  });
+
+  it('should call listener immediately with current value if exists', () => {
+    const signal = createSignal(10);
+    const listener = jest.fn();
+
+    signal(listener);
+    expect(listener).toHaveBeenCalledWith(10);
+  });
+
+  it('should allow unsubscribing', () => {
+    const signal = createSignal<number>();
+    const listener = jest.fn();
+
+    const unsubscribe = signal(listener);
+    signal(42);
+    expect(listener).toHaveBeenCalledTimes(1);
+
+    unsubscribe();
+    signal(43);
+    expect(listener).toHaveBeenCalledTimes(1);
+  });
+
+  it('should allow multiple subscribers', () => {
+    const signal = createSignal<number>();
+    const listener1 = jest.fn();
+    const listener2 = jest.fn();
+
+    signal(listener1);
+    signal(listener2);
+    signal(42);
+
+    expect(listener1).toHaveBeenCalledWith(42);
+    expect(listener2).toHaveBeenCalledWith(42);
+  });
+});
+
+describe('withSignals', () => {
+  it('should combine multiple signals', () => {
+    const signal1 = createSignal<number>();
+    const signal2 = createSignal<string>();
+    const callback = jest.fn();
+
+    withSignals(signal1, signal2)(callback);
+
+    signal1(42);
+    expect(callback).not.toHaveBeenCalled();
+
+    signal2('hello');
+    expect(callback).toHaveBeenCalledWith(42, 'hello');
+  });
+
+  it('should work with initial values', () => {
+    const signal1 = createSignal(42);
+    const signal2 = createSignal('hello');
+    const callback = jest.fn();
+
+    withSignals(signal1, signal2)(callback);
+    expect(callback).toHaveBeenCalledWith(42, 'hello');
+  });
+
+  it('should update when any signal changes', () => {
+    const signal1 = createSignal(42);
+    const signal2 = createSignal('hello');
+    const callback = jest.fn();
+
+    withSignals(signal1, signal2)(callback);
+    signal1(43);
+    
+    expect(callback).toHaveBeenCalledWith(43, 'hello');
+  });
+
+  it('should allow unsubscribing from all signals', () => {
+    const signal1 = createSignal<number>();
+    const signal2 = createSignal<string>();
+    const callback = jest.fn();
+
+    const unsubscribe = withSignals(signal1, signal2)(callback);
+    
+    signal1(42);
+    signal2('hello');
+    expect(callback).toHaveBeenCalledTimes(1);
+
+    unsubscribe();
+    signal1(43);
+    signal2('world');
+    expect(callback).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('derivedSignal', () => {
+  it('should compute derived value', () => {
+    const count = createSignal(5);
+    const multiplier = createSignal(2);
+    
+    const product = derivedSignal(
+      [count, multiplier],
+      (c, m) => c * m
+    );
+
+    expect(product.value).toBe(10);
+  });
+
+  it('should update when source signals change', () => {
+    const count = createSignal(5);
+    const multiplier = createSignal(2);
+    
+    const product = derivedSignal(
+      [count, multiplier],
+      (c, m) => c * m
+    );
+
+    const listener = jest.fn();
+    product(listener);
+
+    count(10);
+    expect(product.value).toBe(20);
+    expect(listener).toHaveBeenCalledWith(20);
+
+    multiplier(3);
+    expect(product.value).toBe(30);
+    expect(listener).toHaveBeenCalledWith(30);
+  });
+
+  it('should handle undefined initial values', () => {
+    const signal1 = createSignal<number>();
+    const signal2 = createSignal<number>();
+    
+    const sum = derivedSignal(
+      [signal1, signal2],
+      (a, b) => a + b
+    );
+
+    expect(sum.value).toBeUndefined();
+
+    signal1(5);
+    expect(sum.value).toBeUndefined();
+
+    signal2(3);
+    expect(sum.value).toBe(8);
+  });
+
+  it('should work with complex derivations', () => {
+    const firstName = createSignal('John');
+    const lastName = createSignal('Doe');
+    const age = createSignal(25);
+
+    const person = derivedSignal(
+      [firstName, lastName, age],
+      (first, last, a) => ({
+        fullName: `${first} ${last}`,
+        age: a,
+        isAdult: a >= 18
+      })
+    );
+
+    expect(person.value).toEqual({
+      fullName: 'John Doe',
+      age: 25,
+      isAdult: true
+    });
+
+    firstName('Jane');
+    expect(person.value).toEqual({
+      fullName: 'Jane Doe',
+      age: 25,
+      isAdult: true
+    });
+  });
+
+  it('should chain derived signals', () => {
+    const base = createSignal(5);
+    const doubled = derivedSignal([base], n => n * 2);
+    const final = derivedSignal([doubled], n => n + 10);
+
+    expect(final.value).toBe(20);
+
+    base(10);
+    expect(doubled.value).toBe(20);
+    expect(final.value).toBe(30);
+  });
+
+  it('should support array transformations', () => {
+    const items = createSignal(['apple', 'banana', 'orange']);
+    const filter = createSignal('an');
+
+    const filtered = derivedSignal(
+      [items, filter],
+      (list, f) => list.filter(item => 
+        item.toLowerCase().includes(f.toLowerCase())
+      )
+    );
+
+    expect(filtered.value).toEqual(['banana', 'orange']);
+
+    filter('ap');
+    expect(filtered.value).toEqual(['apple']);
+
+    items(['grape', 'apple', 'mango']);
+    expect(filtered.value).toEqual(['grape', 'apple']);
+  });
+});
+
+describe('type safety', () => {
+  it('should maintain proper types in derived signals', () => {
+    const numberSignal = createSignal<number>(42);
+    const stringSignal = createSignal<string>('hello');
+
+    // This should type check
+    const derived = derivedSignal(
+      [numberSignal, stringSignal],
+      (num, str) => ({
+        number: num,  // Should be typed as number
+        string: str,  // Should be typed as string
+        combined: `${str}${num}`
+      })
+    );
+
+    expect(derived.value).toEqual({
+      number: 42,
+      string: 'hello',
+      combined: 'hello42'
+    });
+  });
+});

src/utils/signal.ts

@@ -0,0 +1,145 @@
+/**
+ * Enhanced signal types to include value access
+ */
+export interface SignalListener<T> {
+  (event: T): void;
+}
+
+export interface SignalUnsubscriber {
+  (): void;
+}
+
+export interface SignalSubscriber<T> {
+  (listener: SignalListener<T>): SignalUnsubscriber;
+}
+
+export interface SignalDispatcher<T> {
+  (event: T): void;
+}
+
+export interface SignalValue<T> {
+  readonly value: T;
+}
+
+export type Signal<T> = SignalSubscriber<T> & SignalDispatcher<T> & SignalValue<T>;
+export type SignalReturn = SignalUnsubscriber & void;
+
+export function createSignal<T>(): Signal<T>;
+export function createSignal<T>(initialValue: T): Signal<T>;
+export function createSignal<T>(initialValue?: T): Signal<T> {
+  const subscribers = new Set<SignalListener<T>>();
+  let currentValue: T | undefined = initialValue;
+  
+  const signal = ((eventOrListener: T | SignalListener<T>): any => {
+    if (typeof eventOrListener === 'function') {
+      subscribers.add(eventOrListener as SignalListener<T>);
+      // Call the listener immediately with current value if it exists
+      if (currentValue !== undefined) {
+        (eventOrListener as SignalListener<T>)(currentValue);
+      }
+      return () => { subscribers.delete(eventOrListener as SignalListener<T>) };
+    } else {
+      currentValue = eventOrListener;
+      subscribers.forEach(listener => listener(eventOrListener));
+    }
+  }) as Signal<T>;
+
+  // Add value getter
+  Object.defineProperty(signal, 'value', {
+    get: () => currentValue,
+    enumerable: true
+  });
+
+  return signal;
+}
+
+/**
+ * Utility types for handling async compute functions
+ */
+type UnwrapPromise<T> = T extends Promise<infer U> ? U : T;
+
+type ComputeReturnType<Args extends any[], R> = 
+  | ((...args: Args) => R)
+  | ((...args: Args) => Promise<R>);
+
+/**
+ * Utility type to extract the event type from a Signal
+ */
+export type SignalEventType<T> = T extends Signal<infer E> ? E : never;
+
+/**
+ * Creates a new signal that derives its value from other signals
+ * Supports both synchronous and asynchronous compute functions
+ */
+export function derivedSignal<Signals extends Signal<any>[], R>(
+  signals: [...Signals],
+  compute: ComputeReturnType<{ [K in keyof Signals]: SignalEventType<Signals[K]> }, R>
+): Signal<UnwrapPromise<R>> {
+  const derivedSignal = createSignal<UnwrapPromise<R>>();
+  let computeVersion = 0;
+  let isComputing = false;
+  
+  const updateValue = async (version: number, values: { [K in keyof Signals]: SignalEventType<Signals[K]> }) => {
+    if (isComputing) return;
+    isComputing = true;
+    
+    try {
+      const result = compute(...Object.values(values) as { [K in keyof Signals]: SignalEventType<Signals[K]> });
+      const computedValue = result instanceof Promise ? await result : result;
+      
+      // Only update if this is still the latest computation
+      if (version === computeVersion) {
+        derivedSignal(computedValue as UnwrapPromise<R>);
+      }
+    } catch (error) {
+      console.error('Error in derived signal computation:', error);
+    } finally {
+      isComputing = false;
+    }
+  };
+  
+  // Compute initial value if all source signals have values
+  const initialValues = signals.map(s => s.value);
+  if (!initialValues.includes(undefined)) {
+    updateValue(computeVersion, initialValues as { [K in keyof Signals]: SignalEventType<Signals[K]> });
+  }
+  
+  // Handle updates
+  withSignals(...signals)((...args) => {
+    computeVersion++;
+    updateValue(computeVersion, args as { [K in keyof Signals]: SignalEventType<Signals[K]> });
+  });
+  
+  return derivedSignal;
+}
+/**
+ * Takes multiple signals and returns a function that accepts a callback
+ * which will receive the values from those signals
+ */
+export function withSignals<Signals extends Signal<any>[]>(
+  ...signals: Signals
+): <R>(
+  callback: (...values: { [K in keyof Signals]: SignalEventType<Signals[K]> }) => R
+) => SignalUnsubscriber {
+  return (callback) => {
+    const unsubscribers: SignalUnsubscriber[] = [];
+    const values = new Array(signals.length) as { [K in keyof Signals]: SignalEventType<Signals[K]> };
+    let initialized = new Array(signals.length).fill(false);
+    
+    signals.forEach((signal, index) => {
+      const unsubscribe = signal((value) => {
+        values[index] = value as { [K in keyof Signals]: SignalEventType<Signals[K]> }[number];
+        initialized[index] = true;
+        
+        if (initialized.every(Boolean)) {
+          callback(...(values as unknown as { [K in keyof Signals]: SignalEventType<Signals[K]> }));
+        }
+      });
+      unsubscribers.push(unsubscribe);
+    });
+    
+    return () => {
+      unsubscribers.forEach(unsubscribe => unsubscribe());
+    };
+  };
+}

yarn.lock

@@ -1428,11 +1428,6 @@ ansi-styles@^5.0.0:
   resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz"
   integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
 
-antlr4@^4.13.2:
-  version "4.13.2"
-  resolved "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz"
-  integrity sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==
-
 antlr4ts-cli@0.5.0-alpha.4:
   version "0.5.0-alpha.4"
   resolved "https://registry.npmjs.org/antlr4ts-cli/-/antlr4ts-cli-0.5.0-alpha.4.tgz"