diff --git a/main.ts b/main.ts index 600b2a6..b75b588 100644 --- a/main.ts +++ b/main.ts @@ -1,4 +1,4 @@ -import { Menu, MenuItem, Plugin, TAbstractFile, Tasks, TFile } from 'obsidian'; +import { Menu, Plugin, TAbstractFile, Tasks, TFile } from 'obsidian'; import { FilesFileSyncTable } from 'src/fileSyncTable/filesTable'; import { TagsFileSyncTable } from 'src/fileSyncTable/tagsTable'; import { TasksFileSyncTable } from 'src/fileSyncTable/tasksTable'; @@ -37,18 +37,22 @@ export default class SqlSealPlugin extends Plugin { this.registerGlobalApi(); const sqlSeal = new SqlSeal(this.app, false, this.rendererRegistry) // FIXME: set verbose based on the env. - this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) this.sqlSeal = sqlSeal + + await this.sqlSeal.db.connect() + // start syncing when files are loaded this.app.workspace.onLayoutReady(() => { sqlSeal.db.connect().then(() => { + this.fileSync = new SealFileSync(this.app, this) - this.fileSync.addTablePlugin(new FilesFileSyncTable(sqlSeal.db, this.app, sqlSeal.tablesManager, this)) - this.fileSync.addTablePlugin(new TagsFileSyncTable(sqlSeal.db, this.app, sqlSeal.tablesManager)) - this.fileSync.addTablePlugin(new TasksFileSyncTable(sqlSeal.db, this.app, sqlSeal.tablesManager)) + this.fileSync.addTablePlugin(new FilesFileSyncTable(sqlSeal.db, this.app, this)) + this.fileSync.addTablePlugin(new TagsFileSyncTable(sqlSeal.db, this.app)) + this.fileSync.addTablePlugin(new TasksFileSyncTable(sqlSeal.db, this.app)) this.fileSync.init() + this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) }) }) diff --git a/package.json b/package.json index 4d43bf4..8b77254 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ }, "dependencies": { "@ag-grid-community/theming": "^32.3.3", + "@hypersphere/omnibus": "^0.1.6", "@jlongster/sql.js": "^1.6.7", "absurd-sql": "^0.0.54", "ag-grid-community": "^32.3.3", @@ -52,6 +53,7 @@ "markdown-table-ts": "^1.0.3", "node-sql-parser": "^5.3.4", "papaparse": "^5.4.1", - "util": "^0.12.5" + "util": "^0.12.5", + "uuid": "^11.0.4" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5ce4b2..33ce674 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@ag-grid-community/theming': specifier: ^32.3.3 version: 32.3.3 + '@hypersphere/omnibus': + specifier: ^0.1.6 + version: 0.1.6 '@jlongster/sql.js': specifier: ^1.6.7 version: 1.6.7 @@ -44,6 +47,9 @@ importers: util: specifier: ^0.12.5 version: 0.12.5 + uuid: + specifier: ^11.0.4 + version: 11.0.4 devDependencies: '@jest/globals': specifier: ^29.7.0 @@ -719,6 +725,9 @@ packages: resolution: {integrity: sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==} engines: {node: '>=18.18'} + '@hypersphere/omnibus@0.1.6': + resolution: {integrity: sha512-agZuKyhdW0n1JoLYZUuA6Du1QoQn39/LapFgRtbJs7fyRM62C9O2PWISHUCwAKnC1Splshpd8glQgx5pA2zkCg==} + '@iconify-json/simple-icons@1.2.12': resolution: {integrity: sha512-lRNORrIdeLStShxAjN6FgXE1iMkaAgiAHZdP0P0GZecX91FVYW58uZnRSlXLlSx5cxMoELulkAAixybPA2g52g==} @@ -2974,6 +2983,10 @@ packages: util@0.12.5: resolution: {integrity: sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==} + uuid@11.0.4: + resolution: {integrity: sha512-IzL6VtTTYcAhA/oghbFJ1Dkmqev+FpQWnCBaKq/gUluLxliWvO8DPFWfIviRmYbtaavtSQe4WBL++rFjdcGWEg==} + hasBin: true + v8-to-istanbul@9.3.0: resolution: {integrity: sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==} engines: {node: '>=10.12.0'} @@ -3643,6 +3656,8 @@ snapshots: '@humanwhocodes/retry@0.4.1': {} + '@hypersphere/omnibus@0.1.6': {} + '@iconify-json/simple-icons@1.2.12': dependencies: '@iconify/types': 2.0.0 @@ -6313,6 +6328,8 @@ snapshots: is-typed-array: 1.1.13 which-typed-array: 1.1.15 + uuid@11.0.4: {} + v8-to-istanbul@9.3.0: dependencies: '@jridgewell/trace-mapping': 0.3.25 diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 386f4a5..78e31aa 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -1,19 +1,8 @@ import { App, Plugin, TAbstractFile, TFile } from "obsidian"; -import { SqlSeal } from "./sqlSeal"; import { FieldTypes } from "./utils"; -import { TablesManager } from "./dataLoader/collections/tablesManager"; import { sanitise } from "./utils/sanitiseColumn"; import { AFileSyncTable } from "./fileSyncTable/abstractFileSyncTable"; -function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record) { - return { - ...frontmatter, - path: file.path, - name: file.name.replace(/\.[^/.]+$/, ""), - id: file.path - } -} - const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => { const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {} diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts index 6300713..3919c6e 100644 --- a/src/SqlSealCodeblockHandler.ts +++ b/src/SqlSealCodeblockHandler.ts @@ -1,101 +1,116 @@ -import { App, MarkdownPostProcessorContext } from "obsidian" +import { App, MarkdownPostProcessorContext, MarkdownRenderChild, normalizePath, Vault } from "obsidian" import { displayError, displayNotice } from "./ui" -import { resolveFrontmatter } from "./frontmatter" -import { hashString } from "./hash" -import { extractCtes, prefixedIfNotGlobal, updateTables } from "./sqlReparseTables" import { SqlSealDatabase } from "./database" import { Logger } from "./logger" -import { TablesManager } from "./dataLoader/collections/tablesManager" -import { QueryManager } from "./dataLoader/collections/queryManager" -import { parseLanguage, Table, TableWithParentPath } from "./grammar/newParser" +import { parseLanguage, Table } from "./grammar/newParser" import { RendererRegistry, RenderReturn } from "./rendererRegistry" +import { Sync } from "./datamodel/sync" +import { OmnibusRegistrator } from "@hypersphere/omnibus" +import { transformQuery } from "./datamodel/transformer" + +class CodeblockProcessor extends MarkdownRenderChild { + + registrator: OmnibusRegistrator + renderer: RenderReturn + + constructor( + private el: HTMLElement, + private source: string, + private ctx: MarkdownPostProcessorContext, + private rendererRegistry: RendererRegistry, + private db: SqlSealDatabase, + private app: App, + private sync: Sync) { + super(el) + this.registrator = this.sync.getRegistrator() + } + + query: string; + + async onload() { + try { + const results = parseLanguage(this.source) + if (results.tables) { + await this.registerTables(results.tables) + if (!results.queryPart) { + displayNotice(this.el, `Creating tables: ${results.tables.map(t => t.tableName).join(', ')}`) + return + } + } + + this.renderer = this.rendererRegistry.prepareRender(results.intermediateContent)(this.el) + + // FIXME: probably should save the one before transform and perform transform every time we execute it. + this.query = results.queryPart + await this.render() + } catch (e) { + displayError(this.el, e.toString()) + } + } + + onunload() { + this.registrator.offAll() + } + + async render() { + try { + + const registeredTablesForContext = await this.sync.getTablesMappingForContext(this.ctx.sourcePath) + const tranformedQuery = transformQuery(this.query, registeredTablesForContext) + + this.registrator.offAll() + Object.values(registeredTablesForContext).forEach(v => { + this.registrator.on(`change::${v}`, () => { + this.render() + }) + this.registrator.on('file::change::'+this.ctx.sourcePath, () => { + sleep(250).then(() => this.render()) + + }) + }) + + + const file = this.app.vault.getFileByPath(this.ctx.sourcePath) + if (!file) { + return + } + const fileCache = this.app.metadataCache.getFileCache(file) + const { data, columns } = await this.db.select(tranformedQuery, fileCache?.frontmatter ?? {}) + this.renderer.render({ data, columns }) + } catch (e) { + this.renderer.error(e.toString()) + } + } + + async registerTables(tables: Table[]) { + await Promise.all(tables.map((table) => { + const path = this.app.metadataCache.getFirstLinkpathDest(table.fileName, this.ctx.sourcePath) + if (!path) { + throw new Error(`File does not exist: ${table.fileName} (for ${table.tableName}).`) + } + return this.sync.registerTable({ + aliasName: table.tableName, + fileName: path.path, + sourceFile: this.ctx.sourcePath + }) + })) + } +} + export class SqlSealCodeblockHandler { - get globalTables() { - return ['files', 'tags', 'tasks', 'xyz'] // Make this come from SealFileSync and plugins. - } constructor( private readonly app: App, private readonly db: SqlSealDatabase, - private logger: Logger, - private tableManager: TablesManager, - private queryManager: QueryManager, + private sync: Sync, private rendererRegistry: RendererRegistry ) { } - setupTableSignals(tables: Array) { - tables.forEach(t => { - this.tableManager.registerTable(t.tableName, t.fileName, t.parentPath) - }) - } - - setupQuerySignals({ statement, tables }: ReturnType, renderer: RenderReturn, ctx: MarkdownPostProcessorContext, el: Element) { - const frontmatter = resolveFrontmatter(ctx, this.app) - const renderSelect = async () => { - try { - const { data, columns } = await this.db.select(statement, frontmatter ?? {}) - renderer.render({ data, columns }) - } catch (e) { - renderer.error(e.toString()) - } - } - - - const sig = this.queryManager.registerQuery(ctx.docId, tables) - const unsubscribe = sig(() => { - if (!el.isConnected) { - unsubscribe() - return - } - renderSelect() - }) - } - getHandler() { return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => { - const prefix = hashString(ctx.sourcePath) - - let results; - let renderer: RenderReturn; - - try { - await this.db.connect() - // Before we display data, we need to parse query to see if there is query part. - results = parseLanguage(source) - if (results.queryPart) { - // FIXME: this one probably needs both renderer and error api. - renderer = this.rendererRegistry.prepareRender(results.intermediateContent)(el) - } else { - displayNotice(el, `Creating tables: ${results.tables.map(t => t.tableName).join(', ')}`) - } - - const prefixedTables = results.tables.map(t => { - return { - ...t, - tableName: prefixedIfNotGlobal(t.tableName, this.globalTables, prefix), - parentPath: ctx.sourcePath - } - }) satisfies TableWithParentPath[] - - this.setupTableSignals(prefixedTables) - - } catch (e) { - displayError(el, e.toString()) - return - } - - try { - if (results.queryPart) { - const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix) - const ctes = extractCtes(results.queryPart) - const tablesWithoutCtes = tables.filter(t => !ctes.includes(t)) - this.setupQuerySignals({ statement, tables: tablesWithoutCtes }, renderer!, ctx, el) - } - } catch (e) { - renderer!.error(e.toString()) - } - + const processor = new CodeblockProcessor(el, source, ctx, this.rendererRegistry, this.db, this.app, this.sync) + ctx.addChild(processor) } } } \ No newline at end of file diff --git a/src/dataLoader/collections/filesManager.ts b/src/dataLoader/collections/filesManager.ts deleted file mode 100644 index 2a75b01..0000000 --- a/src/dataLoader/collections/filesManager.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Signal, SignalUnsubscriber } from "src/utils/signal"; -import { dataTransformer, DataTransformerOut } from "../dataTransformer"; -import { CSVData, csvFileSignal } from "../csvFile"; -import { App, TFile, Vault } from "obsidian"; -import { isNull } from "lodash"; -import { SqlSealDatabase } from "src/database"; -import { SyncModel } from "src/models/sync"; - -export class FilesManager { - files: Map> = new Map() - inputFiles: Map> = new Map() - unregisters: Array = [] - syncModel: SyncModel - constructor(private vault: Vault, private app: App, private db: SqlSealDatabase) { - this.syncModel = new SyncModel(this.db) - this.vault.on('modify', async (file) => { - if (this.files.has(file.path)) { - this.inputFiles.get(file.path)!(await this.loadFile(file.path)) - } - }) - } - - private async loadFile(url: string, sourcePath?: string) { - let file: TFile | null; - if (sourcePath) { - file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) - } else { - file = this.vault.getFileByPath(url) - } - if (!file) { - return '' - } - const data = await this.vault.cachedRead(file) - return data - } - - getFile(url: string, sourcePath: string = '/') { - return this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) - } - - getFileHandler(url: string, sourcePath?: string) { - let file: TFile | null; - if (sourcePath) { - file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) - } else { - file = this.vault.getFileByPath(url) - } - } - - doesFileExist(url: string, sourcePath: string) { - const file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) - return !isNull(file) - } - - destroy() { - this.unregisters.forEach(u => u()) - } - - private saveManagedRecord(filename: string, checksum: string) { - - } - - getFileSignal(filename: string): Signal { - if (this.files.has(filename)) { - return this.files.get(filename)! - } - const fileSignal = csvFileSignal(filename) - const signal = dataTransformer(fileSignal) - - // Check if it's already in the database, if so, we can just load it again. - this.db - - // FIXME: add here ability to watch changes. - - this.files.set(filename, signal) - this.inputFiles.set(filename, fileSignal) - this.loadFile(filename).then(fileSignal) - - return signal - } - - addUnregister(unreg: SignalUnsubscriber) { - this.unregisters.push(unreg) - } -} \ No newline at end of file diff --git a/src/dataLoader/collections/queryManager.ts b/src/dataLoader/collections/queryManager.ts deleted file mode 100644 index 1b09553..0000000 --- a/src/dataLoader/collections/queryManager.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createSignal, derivedSignal, SignalUnsubscriber, withSignals } from "src/utils/signal"; -import { TablesManager } from "./tablesManager"; - -export class QueryManager { - private registeredQueries: Map = new Map() - constructor(private tablesManager: TablesManager) { } - - registerQuery(fileId: string, tables: Array) { - if (this.registeredQueries.has(fileId)) { - this.registeredQueries.get(fileId)!() - this.registeredQueries.delete(fileId) - } - const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t, true)) - const quertySignal = createSignal() - const unregister = withSignals(...tableSignals)(() => { - quertySignal(Date.now()) - }) - this.registeredQueries.set(fileId, unregister) - return quertySignal - } -} \ No newline at end of file diff --git a/src/dataLoader/collections/tablesManager.ts b/src/dataLoader/collections/tablesManager.ts deleted file mode 100644 index 265ac55..0000000 --- a/src/dataLoader/collections/tablesManager.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { SqlSealDatabase } from "src/database"; -import { FilesManager } from "./filesManager"; -import { createSignal, Signal, SignalUnsubscriber } from "src/utils/signal"; -import { linkTableWithFile } from "../tableSignal"; -import { FileManager } from "obsidian"; -import { isNull } from "lodash"; - -interface Definition { - fileName: string; - unlink: SignalUnsubscriber; -} - - -export class TablesManager { - private tableLinks: Map = new Map() - private tableSignals: Map> = new Map() - constructor(private filesManager: FilesManager, private db: SqlSealDatabase) { - - } - - registerTable(tableName: string, fileName: string, parentPath: string) { - const file = this.filesManager.getFile(fileName, parentPath) - if (isNull(file)) { - throw new Error(`File ${fileName} does not exist`) - } - - if (this.tableLinks.has(tableName)) { - const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)! - if (prevFileName === file.path) { - return - } - unlink() - this.tableLinks.delete(tableName) - } - const resolvedFileName = file.path - const fileSignal = this.filesManager.getFileSignal(resolvedFileName) - const tableSignal = this.getTableSignal(tableName) - - const unlink = linkTableWithFile(fileSignal, tableSignal, tableName, this.db) - this.tableLinks.set(tableName, { - fileName: resolvedFileName, - unlink - }) - } - - getTableSignal(tableName: string, failOnUndefined: boolean = false) { - if (!this.tableSignals.has(tableName)) { - if (failOnUndefined) { - throw new Error(`${tableName} does not exist`) - } - this.tableSignals.set(tableName, createSignal()) - } - return this.tableSignals.get(tableName)! - } -} \ No newline at end of file diff --git a/src/dataLoader/csvFile.ts b/src/dataLoader/csvFile.ts deleted file mode 100644 index c8000d5..0000000 --- a/src/dataLoader/csvFile.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { TFile } from "obsidian"; -import { createSignal, Signal } from "src/utils/signal"; - -export type CSVData = string - -export const csvFileSignal = (_file: string): Signal => { - const sig = createSignal() - return sig -} \ No newline at end of file diff --git a/src/dataLoader/dataTransformer.test.ts b/src/dataLoader/dataTransformer.test.ts deleted file mode 100644 index ff967d9..0000000 --- a/src/dataLoader/dataTransformer.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, it, expect, jest } from '@jest/globals' -import { createSignal } from '../utils/signal' -import { dataTransformer } from './dataTransformer' - -describe('Data Transformer', () => { - it('should properly transform basic file', () => { - const s = createSignal() - const dt = dataTransformer(s) - s(`a,b,c -1,2,3 -4,5,6`) - expect(dt.value.data).toHaveLength(2) - expect(dt.value.data[0]).toEqual({ - a: 1, - b: 2, - c: 3 - }) - - expect(dt.value.types).toEqual({ - a: 'INTEGER', - b: 'INTEGER', - c: 'INTEGER' - }) - }) - - it('should properly transform incorrect keys', () => { - const s = createSignal() - const dt = dataTransformer(s) - s(`BEGIN,PLAN,QUERY,RAISE,beak size (mm) -1.5,hello world,343.423,22.34,2`) - expect(dt.value.types).toEqual({ - 'begin_': 'REAL', - 'plan_': 'TEXT', - 'query_': 'REAL', - 'raise_': 'REAL', - 'beak_size__mm_': 'INTEGER' - }) - }) -}) \ No newline at end of file diff --git a/src/dataLoader/dataTransformer.ts b/src/dataLoader/dataTransformer.ts deleted file mode 100644 index 0d367ff..0000000 --- a/src/dataLoader/dataTransformer.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { derivedSignal, Signal, SignalEventType } from "../utils/signal" -import { CSVData } from "./csvFile" -import { parse } from 'papaparse' -import { toTypeStatements } from "../utils" -import { sanitise } from "../utils/sanitiseColumn" -import { TFile } from "obsidian" - - -export const parseData = (csvData: string) => { - const parsed = parse>(csvData, { - header: true, - dynamicTyping: false, - skipEmptyLines: true, - transformHeader: sanitise - }) - return parsed -} - - -export const dataTransformer = (s: Signal) => { - const sig = derivedSignal([s], (data) => { - try { - const parsed = parseData(data) - const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data) - return typeStatements - } catch (e) { - console.error(e); - return { - data: [], - types: {} - } - } - }) - - return sig -} - -export type DataTransformerOut = SignalEventType> diff --git a/src/dataLoader/tableSignal.ts b/src/dataLoader/tableSignal.ts deleted file mode 100644 index e3641d9..0000000 --- a/src/dataLoader/tableSignal.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Signal } from "src/utils/signal"; -import { DataTransformerOut } from "./dataTransformer"; -import { SqlSealDatabase } from "src/database"; -import { FieldTypes } from "src/utils"; - -export const linkTableWithFile = (dataSig: Signal, tableSignal: Signal, tableName: string, db: SqlSealDatabase) => { - return dataSig(({ data, types }) => { - - // Check if the columns are exactly the same. If not, delete and reinstantiate the table. - - // For now always remove and reinstantiate the table - db.dropTable(tableName) - - // compute the types of the keys - const columns = Object.entries(types).map(([key, value]) => ({ - name: key, - type: value as FieldTypes - })); - - // create table again - db.createTableClean(tableName, columns) - - // Load all the data - db.insertData(tableName, data) - - // Here we do the actual update of the table. If it succeeds, we return it with current date to indicate sync time. - tableSignal(Date.now()) - }) -} diff --git a/src/database-worker.ts b/src/database-worker.ts index e277ab4..c04d7b8 100644 --- a/src/database-worker.ts +++ b/src/database-worker.ts @@ -112,20 +112,23 @@ export class WorkerDatabase { }) } - async createTable(tableName: string, fields: Record) { + async createTable(tableName: string, fields: Record, noDrop: boolean = false) { const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type]) const uniqueFields = [...new Map(transformedFiels.map(item => [item[0], item])).values()] const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`) // FIXME: probably use schema generator, for now create with hardcoded fields - await this.dropTable(tableName) + if (!noDrop) { + await this.dropTable(tableName) + } const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} ( ${sqlFields.join(', ')} );` this.db.prepare(createSQL).run() - await this.clearTable(tableName) - + if (!noDrop) { + await this.clearTable(tableName) + } // Dropping data. } @@ -154,10 +157,10 @@ export class WorkerDatabase { } } - async updateData(tableName: string, data: Array>) { + async updateData(tableName: string, data: Array>, matchKey: string = 'id') { const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); data.forEach((d: Record) => { - const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`) + const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE ${matchKey} = @${matchKey}`) stmt.run(recordToBindParams(d)) }) } diff --git a/src/database.ts b/src/database.ts index 1893e44..46f5d64 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,8 +1,5 @@ import { App } from "obsidian" import { FieldTypes, toTypeStatements } from "./utils" -import { sanitise } from "./utils/sanitiseColumn" -import initSqlJs, { BindParams, Database, Statement } from 'sql.js' -import wasmBinary from '../node_modules/sql.js/dist/sql-wasm.wasm' import * as Comlink from 'comlink' import workerCode from 'virtual:worker-code' import { WorkerDatabase } from "./database-worker" @@ -96,21 +93,6 @@ export class SqlSealDatabase { return schema } - // async addNewColumns(name: string, data: Array>) { - // const schema = await this.getSchema(data) - // const currentSchema = this.toObjectsArray(this.db.prepare(`PRAGMA table_info(${name})`)) - // const currentFields = currentSchema.map((f: any) => f.name) - // const newFields = Object.keys(schema).filter(f => !currentFields.includes(f)) - - // if (newFields.length === 0) { - // return - // } - - // const alter = this.db.prepare(`ALTER TABLE ${name} ADD - // COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`) - // alter.run() - // } - async updateData(name: string, data: Array>) { return this.db.updateData(name, data) } @@ -135,8 +117,8 @@ export class SqlSealDatabase { await this.createTable(name, fieldsToRecord) } - async createTable(name: string, fields: Record) { - await this.db.createTable(name, fields) + async createTable(name: string, fields: Record, noDrop?: boolean) { + await this.db.createTable(name, fields, noDrop) } async getSchema(data: Array>) { const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); diff --git a/src/datamodel/hasher.ts b/src/datamodel/hasher.ts new file mode 100644 index 0000000..f892bf6 --- /dev/null +++ b/src/datamodel/hasher.ts @@ -0,0 +1,54 @@ +/** + * A utility class for generating consistent hashes from filepaths in browser environments + */ +export class FilepathHasher { + /** + * Creates a SHA-256 hash of a filepath + * @param {string} filepath - The filepath to hash + * @returns {Promise} The hex-encoded hash + */ + static async sha256(filepath: string) { + // Normalize the filepath to ensure consistent hashing across platforms + const normalizedPath = filepath.replace(/\\/g, '/').toLowerCase(); + + // Convert string to array buffer + const encoder = new TextEncoder(); + const data = encoder.encode(normalizedPath); + + // Generate hash using browser's crypto API + const hashBuffer = await window.crypto.subtle.digest('SHA-256', data); + + // Convert to hex string + return Array.from(new Uint8Array(hashBuffer)) + .map(b => b.toString(16).padStart(2, '0')) + .join(''); + } + + /** + * Creates a shorter hash (first 8 characters of SHA-256) + * @param {string} filepath - The filepath to hash + * @returns {Promise} The truncated hex-encoded hash + */ + static async shortHash(filepath: string) { + const fullHash = await this.sha256(filepath); + return fullHash.slice(0, 8); + } + + /** + * Creates a base64 encoded hash of a filepath + * @param {string} filepath - The filepath to hash + * @returns {Promise} The base64-encoded hash + */ + static async base64Hash(filepath: string) { + const normalizedPath = filepath.replace(/\\/g, '/').toLowerCase(); + const encoder = new TextEncoder(); + const data = encoder.encode(normalizedPath); + const hashBuffer = await window.crypto.subtle.digest('SHA-256', data); + + // Convert ArrayBuffer to Base64 + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashString = String.fromCharCode.apply(null, hashArray); + return btoa(hashString); + } + } + \ No newline at end of file diff --git a/src/datamodel/repository/abstractRepository.ts b/src/datamodel/repository/abstractRepository.ts new file mode 100644 index 0000000..a38132a --- /dev/null +++ b/src/datamodel/repository/abstractRepository.ts @@ -0,0 +1,5 @@ +import { SqlSealDatabase } from "src/database"; + +export abstract class Repository { + constructor(protected readonly db: SqlSealDatabase) { } +} \ No newline at end of file diff --git a/src/datamodel/repository/fileLog.ts b/src/datamodel/repository/fileLog.ts new file mode 100644 index 0000000..08114d9 --- /dev/null +++ b/src/datamodel/repository/fileLog.ts @@ -0,0 +1,65 @@ +import { Repository } from "./abstractRepository"; +import { v4 as uuidv4 } from "uuid"; + +export interface FileLog { + id: string, + table_name: string, + file_name: string, + created_at: string, + updated_at: string, + file_hash: string +} + +export class FileLogRepository extends Repository { + + async init() { + await this.createTable() + } + + private async createTable() { + await this.db.createTable('file_log', { + 'id': 'TEXT', + 'table_name': 'TEXT', + 'file_name': 'TEXT', + 'created_at': 'TEXT', + 'updated_at': 'TEXT', + 'file_hash': 'TEXT' + }, true) + } + + async insert(log: Omit) { + const now = Date.now() + await this.db.insertData('file_log', [{ + id: uuidv4(), + table_name: log.table_name, + file_name: log.file_name, + file_hash: log.file_hash, + created_at: now, + updated_at: now + }]) + } + + async getByFilename(filename: string) { + const { data } = await this.db.select('SELECT * FROM file_log WHERE file_name = @filename', { + filename: filename + }) + + if (!data.length) { + return null + } + return data[0] as unknown as FileLog + } + + async getAll() { + const { data } = await this.db.select('SELECT * FROM file_log', {}) + return data as unknown[] as FileLog[]; + } + + async updateHash(tableName: string, newHash: string) { + await this.db.db.updateData('file_log', [{ + table_name: tableName, + file_hash: newHash + }], 'table_name') + + } +} \ No newline at end of file diff --git a/src/datamodel/repository/tableMapLog.ts b/src/datamodel/repository/tableMapLog.ts new file mode 100644 index 0000000..3419935 --- /dev/null +++ b/src/datamodel/repository/tableMapLog.ts @@ -0,0 +1,86 @@ +import { Repository } from "./abstractRepository"; +import { v4 as uuid4 } from "uuid"; + +export interface TableMapLog { + id: string; + table_name: string; + alias_name: string; + source_file_name: string; + created_at: string; + updated_at: string; +} + +export class TableMapLogRepository extends Repository { + + async init() { + await this.createTable() + } + + async deleteMapping(id: string) { + this.db.deleteData('table_map_log', [{ id: id }], 'id') + } + + private async createTable() { + // FIXME: add autoincrement index here to make it easier to manage. + await this.db.createTable('table_map_log', { + 'id': 'TEXT', + 'table_name': 'TEXT', + 'alias_name': 'TEXT', + 'source_file_name': 'TEXT', + 'created_at': 'TEXT', + 'updated_at': 'TEXT' + }, true) + } + + async insert(log: Omit) { + const now = Date.now() + await this.db.insertData('table_map_log', [{ + id: uuid4(), + table_name: log.table_name, + alias_name: log.alias_name, + source_file_name: log.source_file_name, + created_at: now, + updated_at: now + }]) + } + + async getAll() { + const { data } = await this.db.select('SELECT * FROM table_map_log', {}) + return data as unknown as TableMapLog[] + } + + async getByAlias(sourceFileName: string, aliasName: string) { + const { data } = await this.db.select(`SELECT * FROM table_map_log + WHERE source_file_name=@source_file_name + AND alias_name=@alias_name`, { + 'source_file_name': sourceFileName, + 'alias_name': aliasName + }) + if (!data || data.length < 0) { + return null + } + return data[0] as unknown as TableMapLog + } + + async getByTableName(tableName: string) { + const { data } = await this.db.select(`SELECT * FROM table_map_log + WHERE table_name = @table_name`, { + table_name: tableName + }) + if (!data) { + return [] + } + return data as unknown[] as TableMapLog[] + } + + + async getByContext(sourceFileName: string) { + const { data } = await this.db.select(`SELECT * FROM table_map_log + WHERE source_file_name=@source_file_name + `, { source_file_name: sourceFileName }) + if (!data) { + return [] + } + return data + } +} \ No newline at end of file diff --git a/src/datamodel/sync.ts b/src/datamodel/sync.ts new file mode 100644 index 0000000..0b155b9 --- /dev/null +++ b/src/datamodel/sync.ts @@ -0,0 +1,199 @@ +import { SqlSealDatabase } from "src/database"; +import { FileLog, FileLogRepository } from "./repository/fileLog"; +import { TableMapLogRepository } from "./repository/tableMapLog"; +import { TAbstractFile, TFile, Vault } from "obsidian"; +import { parse } from "papaparse"; +import { sanitise } from "src/utils/sanitiseColumn"; +import { FieldTypes, toTypeStatements } from "src/utils"; +import { FilepathHasher } from "./hasher"; +import { Omnibus } from "@hypersphere/omnibus"; + + +interface TableRegistration { + fileName: string; + aliasName: string; + sourceFile: string; +} + +export class Sync { + private fileLog: FileLogRepository; + private tableMapLog: TableMapLogRepository; + private bus = new Omnibus() + + private tableMaps = new Map() + + constructor( + private readonly db: SqlSealDatabase, + private readonly vault: Vault + ) { + + } + + async updateTableMaps() { + const data = await this.fileLog.getAll() + this.tableMaps.clear() + data.forEach(log => { + this.tableMaps.set(log.file_name, log) + }) + } + + async init() { + await this.db.connect() + this.fileLog = new FileLogRepository(this.db) + await this.fileLog.init() + + this.tableMapLog = new TableMapLogRepository(this.db) + await this.tableMapLog.init() + + const fileLogs = await this.fileLog.getAll() + for (const log of fileLogs) { + await this.syncFileByName(log.file_name) + } + + await this.updateTableMaps() + + // START SYNCING + this.startSync() + } + + async syncFileByName(fileName: string) { + const file = this.vault.getFileByPath(fileName) + if (!file) { + return + } + await this.syncFile(file) + } + + async syncFile(file: TFile) { + if (!this.tableMaps.has(file.path)) { + this.bus.trigger('file::change::'+file.path) + return; + } + + const entry = this.tableMaps.get(file.path)! + if (entry.file_hash !== file.stat.mtime.toString()) { + const data = await this.vault.cachedRead(file) + // TODO: PROBABLY SHOULD BE EXTRACTED SOMEWHERE FROM HERE later. + const parsed = parse>(data, { + header: true, + dynamicTyping: false, + skipEmptyLines: true, + transformHeader: sanitise + }) + const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data) + const columns = Object.entries(typeStatements.types).map(([key, value]) => ({ + name: key, + type: value as FieldTypes + })); + const tableName = entry.table_name + await this.db.createTableClean(tableName, columns) + await this.db.insertData(tableName, typeStatements.data) + await this.fileLog.updateHash(tableName, file.stat.mtime.toString()) + this.bus.trigger('change::' + tableName) + + } + } + + startSync() { + this.vault.on('modify', (file: TAbstractFile) => { + if (file instanceof TFile) { + this.syncFile(file) + } + }) + } + + async registerFile(log: Omit) { + await this.fileLog.insert(log) + await this.updateTableMaps() + } + + async getTablesMappingForContext(sourceFileName: string) { + const tables = await this.tableMapLog.getByContext(sourceFileName) + const map = tables.reduce((acc, t) => ({ + ...acc, + [t.alias_name as string]: t.table_name + }), {}) + + return { + ...map, + files: 'files', + tasks: 'tasks', + tags: 'tags' + } + } + + async generateTableName(fileName: string) { + const hash = await FilepathHasher.sha256(fileName) + return `file_${hash}` + } + + async registerTable(reg: TableRegistration) { + const existingFileLog = await this.fileLog.getByFilename(reg.fileName) + let fileTableName: string; + if (!existingFileLog) { + // We need to create new one. + const tableName = await this.generateTableName(reg.fileName) + await this.registerFile({ + file_name: reg.fileName, + table_name: tableName, + file_hash: '' + }) + fileTableName = tableName + // INITIAL SYNC + await this.syncFileByName(reg.fileName) + } else { + fileTableName = existingFileLog.table_name + } + + const existingTableLog = await this.tableMapLog.getByAlias(reg.sourceFile, reg.aliasName) + if (!existingTableLog) { + // Create new one + await this.tableMapLog.insert({ + alias_name: reg.aliasName, + source_file_name: reg.sourceFile, + table_name: fileTableName, + }) + } else { + // Already exists, doing nothing. + // Check if it is the same mapping + if (existingTableLog.table_name != fileTableName) { + await this.tableMapLog.deleteMapping(existingTableLog.id) // FIXME: might need to remove event too + await this.tableMapLog.insert({ + alias_name: reg.aliasName, + table_name: fileTableName, + source_file_name: reg.sourceFile + }) + } + } + } + + getRegistrator() { + return this.bus.getRegistrator() + } + + async getEventNameForAlias(sourceFileName: string, aliasName: string) { + const log = await this.tableMapLog.getByAlias(sourceFileName, aliasName) + if (!log) { + throw new Error(`${aliasName} does not exist for ${sourceFileName}`) + } + return `change::${log.table_name}` + } + + // async onFileChange(file: TFile) { + // const path = file.path + // const hash = file.stat.mtime + // // Checking if the file is registered + // const reg = await this.fileLog.getByFilename(path) + // if (reg) { + // // Need to syncronise it. + // // Need to save update in the database. + // // Need to trigger all related files. + // const allRelated = this.tableMapLog.getByTableName(reg.tableName) + // for (const related of allRelated) { + // this.bus.bus.trigger('') + // } + // } + + // // Checking if the file file is in the map. + // } +} \ No newline at end of file diff --git a/src/datamodel/transformer.ts b/src/datamodel/transformer.ts new file mode 100644 index 0000000..a064df2 --- /dev/null +++ b/src/datamodel/transformer.ts @@ -0,0 +1,143 @@ +import { Parser } from 'node-sql-parser'; + +export function transformQuery(query: string, tablesAliases: Record): string { + const parser = new Parser(); + + // Parse the query into an AST + const ast = parser.astify(query, { + database: 'Sqlite' + }); + + // Handle both single queries and multiple queries (like UNION) + const queries = Array.isArray(ast) ? ast : [ast]; + + // Process each query in the array + queries.forEach(queryAst => { + transformQueryPart(queryAst, tablesAliases); + }); + + // Convert back to SQL + return parser.sqlify(ast, { database: 'Sqlite' }); +} + +function transformQueryPart(ast: any, tablesAliases: Record): void { + if (!ast || typeof ast !== 'object') return; + + // Handle FROM clause + if (ast.from) { + ast.from = ast.from.map((fromItem: any) => { + // Handle regular table references + if (fromItem.table && tablesAliases[fromItem.table]) { + fromItem.table = tablesAliases[fromItem.table]; + } + + // Handle function expressions in FROM clause (like json_each) + if (fromItem.expr) { + transformExpression(fromItem.expr, tablesAliases); + } + + // Handle nested queries in FROM + if (fromItem.stmt) { + transformQueryPart(fromItem.stmt, tablesAliases); + } + + return fromItem; + }); + } + + // Handle WITH clause (CTEs) + if (ast.with && Array.isArray(ast.with)) { + ast.with.forEach((withItem: any) => { + if (withItem.stmt) { + transformQueryPart(withItem.stmt.ast, tablesAliases); + } + }); + } + + // Handle legacy WITH clause structure + if (ast.with && ast.with.ctes && Array.isArray(ast.with.ctes)) { + ast.with.ctes.forEach((withItem: any) => { + if (withItem.stmt) { + transformQueryPart(withItem.stmt, tablesAliases); + } + }); + } + + // Handle JOINs + if (ast.join) { + ast.join = ast.join.map((joinItem: any) => { + if (joinItem.table && joinItem.table.table && tablesAliases[joinItem.table.table]) { + joinItem.table.table = tablesAliases[joinItem.table.table]; + } + if (joinItem.on) { + transformExpression(joinItem.on, tablesAliases); + } + return joinItem; + }); + } + + // Handle WHERE clause + if (ast.where) { + transformExpression(ast.where, tablesAliases); + } + + // Handle UNION queries + if (ast.union) { + transformQueryPart(ast.union[0], tablesAliases); + transformQueryPart(ast.union[1], tablesAliases); + } + + // Handle SELECT columns + if (ast.columns) { + ast.columns.forEach((column: any) => { + transformExpression(column.expr, tablesAliases); + }); + } +} + +function transformExpression(expr: any, tablesAliases: Record): void { + if (!expr || typeof expr !== 'object') return; + + // Handle column references + if (expr.type === 'column_ref' && expr.table && tablesAliases[expr.table]) { + expr.table = tablesAliases[expr.table]; + } + + // Handle function calls + if (expr.type === 'function') { + if (expr.args && Array.isArray(expr.args)) { + expr.args.forEach((arg: any) => { + if (arg.type === 'column_ref' && arg.table && tablesAliases[arg.table]) { + arg.table = tablesAliases[arg.table]; + } + transformExpression(arg, tablesAliases); + }); + } + } + + // Handle binary expressions + if (expr.type === 'binary_expr') { + transformExpression(expr.left, tablesAliases); + transformExpression(expr.right, tablesAliases); + } + + // Handle CASE expressions + if (expr.type === 'case') { + if (expr.args && Array.isArray(expr.args)) { + expr.args.forEach((arg: any) => transformExpression(arg, tablesAliases)); + } + } + + // Recursively process other potential expressions + Object.keys(expr).forEach(key => { + if (Array.isArray(expr[key])) { + expr[key].forEach((item: any) => { + if (item && typeof item === 'object') { + transformExpression(item, tablesAliases); + } + }); + } else if (expr[key] && typeof expr[key] === 'object') { + transformExpression(expr[key], tablesAliases); + } + }); +} diff --git a/src/fileSyncTable/abstractFileSyncTable.ts b/src/fileSyncTable/abstractFileSyncTable.ts index 1be617e..4d3fedf 100644 --- a/src/fileSyncTable/abstractFileSyncTable.ts +++ b/src/fileSyncTable/abstractFileSyncTable.ts @@ -1,12 +1,10 @@ import { App, TFile } from "obsidian"; import { SqlSealDatabase } from "src/database"; -import { TablesManager } from "src/dataLoader/collections/tablesManager"; export abstract class AFileSyncTable { constructor( protected readonly db: SqlSealDatabase, - protected readonly app: App, - protected tableManager: TablesManager + protected readonly app: App ) { } diff --git a/src/fileSyncTable/filesTable.ts b/src/fileSyncTable/filesTable.ts index 98cd9af..2d6384f 100644 --- a/src/fileSyncTable/filesTable.ts +++ b/src/fileSyncTable/filesTable.ts @@ -1,8 +1,7 @@ -import { App, getAllTags, Plugin, TAbstractFile, TFile } from "obsidian"; +import { App, Plugin, TAbstractFile, TFile } from "obsidian"; import { AFileSyncTable } from "./abstractFileSyncTable"; import { sanitise } from "src/utils/sanitiseColumn"; import { SqlSealDatabase } from "src/database"; -import { TablesManager } from "src/dataLoader/collections/tablesManager"; import { predictType } from "src/utils"; @@ -27,19 +26,17 @@ function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record> shouldPerformBulkInsert = true; - constructor(db: SqlSealDatabase, app: App, tableManager: TablesManager, private readonly plugin: Plugin) { - super(db, app, tableManager) + constructor(db: SqlSealDatabase, app: App, private readonly plugin: Plugin) { + super(db, app) } async onFileModify(file: TFile): Promise { // we need to update the row const frontmatter = await extractFrontmatterFromFile(file, this.plugin) await this.db.updateData('files', [fileData(file, frontmatter)]) - this.tableManager.getTableSignal('files')(Date.now()) await sleep(1000) // TO DELAY OTHER PLUGINS, UGLY BUT WORKS. } async onFileDelete(path: string): Promise { await this.db.deleteData('files', [{ id: path }]) - this.tableManager.getTableSignal('files')(Date.now()) } async onFileCreate(file: TFile): Promise { @@ -49,7 +46,6 @@ export class FilesFileSyncTable extends AFileSyncTable { // TODO: Check if there are new columns. If there are, we update the database. await this.db.insertData('files', [fileData(file, frontmatter)]) - this.tableManager.getTableSignal('files')(Date.now()) // FIXME: should we be waiting here? await sleep(1000) @@ -60,7 +56,6 @@ export class FilesFileSyncTable extends AFileSyncTable { const data = await Promise.all(files.map(async f => fileData(f, await extractFrontmatterFromFile(f, this.plugin)))) const schema = await this.db.createTableWithData('files', data) this.currentSchema = schema - this.tableManager.getTableSignal('files')(Date.now()) } diff --git a/src/fileSyncTable/tagsTable.ts b/src/fileSyncTable/tagsTable.ts index d04c16e..c516e66 100644 --- a/src/fileSyncTable/tagsTable.ts +++ b/src/fileSyncTable/tagsTable.ts @@ -8,13 +8,11 @@ export class TagsFileSyncTable extends AFileSyncTable { } async onFileDelete(path: string): Promise { await this.db.deleteData('tags', [{ fileId: path }], 'fileId') - this.tableManager.getTableSignal('tags')(Date.now()) } async onFileCreate(file: TFile): Promise { const tags = await this.getFileTags(file) this.db.insertData('tags', tags) - this.tableManager.getTableSignal('tags')(Date.now()) } private async getFileTags(file: TFile) { @@ -38,6 +36,5 @@ export class TagsFileSyncTable extends AFileSyncTable { 'tag': 'TEXT', 'fileId': 'TEXT' }) - this.tableManager.getTableSignal('tags')(Date.now()) } } \ No newline at end of file diff --git a/src/fileSyncTable/tasksTable.ts b/src/fileSyncTable/tasksTable.ts index 0f1a6e5..669b5c7 100644 --- a/src/fileSyncTable/tasksTable.ts +++ b/src/fileSyncTable/tasksTable.ts @@ -8,13 +8,11 @@ export class TasksFileSyncTable extends AFileSyncTable { } async onFileDelete(path: string): Promise { await this.db.deleteData('tasks', [{ filePath: path }], 'filePath') - this.tableManager.getTableSignal('tasks')(Date.now()) } async onFileCreate(file: TFile): Promise { const tasks = await this.getFileTags(file) this.db.insertData('tasks', tasks) - this.tableManager.getTableSignal('tasks')(Date.now()) } async getFileTags(file: TFile) { @@ -52,6 +50,5 @@ export class TasksFileSyncTable extends AFileSyncTable { 'completed': 'INTEGER', 'filePath': 'TEXT' }) - this.tableManager.getTableSignal('tasks')(Date.now()) } } \ No newline at end of file diff --git a/src/frontmatter.ts b/src/frontmatter.ts deleted file mode 100644 index f3b7122..0000000 --- a/src/frontmatter.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { App, MarkdownPostProcessorContext } from "obsidian" - -export const resolveFrontmatter = (ctx: Pick, app: App) => { - if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) { - return ctx.frontmatter as Record - } - const file = app.vault.getFileByPath(ctx.sourcePath) - if (!file) { - return null - } - return app.metadataCache.getFileCache(file)?.frontmatter -} \ No newline at end of file diff --git a/src/hash.ts b/src/hash.ts deleted file mode 100644 index 5169661..0000000 --- a/src/hash.ts +++ /dev/null @@ -1,118 +0,0 @@ - -export function hashString(inputString: string): string { - // Function to convert a number to alphanumeric character - function toAlphanumeric(num: number): string { - const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - return chars[num % chars.length]; - } - - // Function to generate SHA-256 hash - function sha256(input: string): string { - // Simple SHA-256 implementation - function sha256Internal(input: string): string { - function rightRotate(value: number, shift: number): number { - return (value >>> shift) | (value << (32 - shift)); - } - - function toHexString(num: number): string { - let hex = ''; - for (let i = 0; i < 8; i++) { - hex += ((num >>> (24 - i * 4)) & 0xf).toString(16); - } - return hex; - } - - const words: number[] = []; - for (let i = 0; i < input.length * 8; i += 8) { - words[i >> 5] |= (input.charCodeAt(i / 8) & 0xff) << (24 - i % 32); - } - words[input.length >> 2] |= 0x80 << (24 - (input.length % 4) * 8); - words[((input.length + 64 >> 9) << 4) + 15] = input.length * 8; - - let a = 0x6a09e667; - let b = 0xbb67ae85; - let c = 0x3c6ef372; - let d = 0xa54ff53a; - let e = 0x510e527f; - let f = 0x9b05688c; - let g = 0x1f83d9ab; - let h = 0x5be0cd19; - - for (let i = 0; i < words.length; i += 16) { - let aa = a; - let bb = b; - let cc = c; - let dd = d; - let ee = e; - let ff = f; - let gg = g; - let hh = h; - - for (let j = 0; j < 64; j++) { - let T1 = hh + ((ee & ff) | (~ee & gg)) + words[i + j] + K[j]; - T1 = T1 | 0; - let T2 = rightRotate(e, 6) ^ rightRotate(e, 11) ^ rightRotate(e, 25); - T2 = T2 | 0; - let T3 = (e & f) ^ (~e & g); - T3 = T3 | 0; - let T4 = h + T1 + T2 + T3 + H[j]; - T4 = T4 | 0; - - h = g; - g = f; - f = e; - e = d + T4; - e = e | 0; - d = c; - c = b; - b = a; - a = T4 + T1; - a = a | 0; - } - - a = a + aa; - b = b + bb; - c = c + cc; - d = d + dd; - e = e + ee; - f = f + ff; - g = g + gg; - h = h + hh; - } - - return toHexString(a) + toHexString(b) + toHexString(c) + toHexString(d) + - toHexString(e) + toHexString(f) + toHexString(g) + toHexString(h); - } - - // Convert the SHA-256 hash to alphanumeric characters - function hashToAlphanumeric(hash: string): string { - return hash.replace(/[^\w]/g, ''); - } - - return hashToAlphanumeric(sha256Internal(input)); - } - - // Constants for SHA-256 - const K = [ - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, - 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, - 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, - 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, - 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, - 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, - 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 - ]; - - const H = [ - 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 - ]; - - return sha256(inputString); -} - - - -export const generatePrefix = (prefix: string, tableName: string) => { - return `TTT${prefix}_${tableName}` -} \ No newline at end of file diff --git a/src/models/sync.ts b/src/models/sync.ts deleted file mode 100644 index 14c25ca..0000000 --- a/src/models/sync.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { SqlSealDatabase } from "src/database"; - - -export interface SyncEntry { - filename: string; - url: string; - name: string; - syncedAt: string; - metadata: any; -} - - -export class SyncModel { - tableName = 'sqlseal_sync' - constructor(private db: SqlSealDatabase) { - this.db.connect().then(() => - this.createTableIfNotExists() - ) - } - - createTableIfNotExists() { - this.db.createTable(this.tableName, { - 'filename': 'TEXT', - 'checksum': 'TEXT', - 'syncedAt': 'TEXT', - }) - } - - registerSync(filename: string, checksum: string) { - this.db.insertData(this.tableName, [{ - filename, - checksum, - syncedAt: Date.now(), - }]) - } - - async getSync(filename: string) { - const { data } = await this.db.select(`SELECT * - FROM sqlseal_sync - WHERE filename = @filename`, - { - filename: filename - }) - - if (data) { - return data[0] as unknown as SyncEntry - } - return null - } - - removeSync(filename: string) { - // FIXME: do not use exposed db, instead implement "exec" method inside the file. - this.db.db.prepare('DELETE FROM sqlseal_sync WHERE filename = :filename AND name = :name').run({ - filename: filename, - name: tableName - }) - } - -} \ No newline at end of file diff --git a/src/settings/SQLSealSettingsTab.ts b/src/settings/SQLSealSettingsTab.ts index 7196e85..438ab54 100644 --- a/src/settings/SQLSealSettingsTab.ts +++ b/src/settings/SQLSealSettingsTab.ts @@ -26,10 +26,6 @@ export class SQLSealSettingsTab extends PluginSettingTab { const {containerEl} = this; containerEl.empty(); - // CSV Viewer section - // containerEl.createEl('h3', { text: 'CSV Viewer' }); - - new Setting(containerEl) .setName('Enable CSV Viewer') .setDesc('Enables CSV files in your vault and adds ability to display them in a grid.') diff --git a/src/sql/parser.test.ts b/src/sql/parser.test.ts new file mode 100644 index 0000000..c21e76c --- /dev/null +++ b/src/sql/parser.test.ts @@ -0,0 +1,198 @@ +import { describe, beforeEach, it, expect } from '@jest/globals' + +import { SQLContextTransformer, TransformationConfig } from './parser'; + +describe('SQLContextTransformer', () => { + let transformer: SQLContextTransformer; + + beforeEach(() => { + const config: TransformationConfig = { + excludedTables: new Set(['metadata']), + customTransforms: { + 'special': (contextId: string) => `${contextId}_special_${new Date().getFullYear()}` + } + }; + transformer = new SQLContextTransformer(config); + }); + + const normalizeSQL = (sql: string): string => { + return sql.replace(/\s+/g, ' ').trim(); + }; + + describe('Basic Transformations', () => { + it('should transform simple SELECT queries', () => { + const input = 'SELECT * FROM users'; + const expected = 'SELECT * FROM `ctx1_users`'; + expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) + .toBe(normalizeSQL(expected)); + }); + + it('should not transform excluded tables', () => { + const input = 'SELECT * FROM metadata'; + const expected = 'SELECT * FROM `metadata`'; + expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) + .toBe(normalizeSQL(expected)); + }); + + it('should apply custom transformations', () => { + const input = 'SELECT * FROM special'; + const expected = `SELECT * FROM \`ctx1_special_${new Date().getFullYear()}\``; + expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) + .toBe(normalizeSQL(expected)); + }); + }); + + describe('Complex Queries - CTEs', () => { + it('should handle single CTE', () => { + const input = ` + WITH recurring_users AS ( + SELECT user_id, COUNT(*) as visits + FROM visits + GROUP BY user_id + HAVING COUNT(*) > 5 + ) + SELECT * FROM recurring_users + JOIN users ON users.id = recurring_users.user_id + `; + const expected = ` + WITH \`recurring_users\` AS + (SELECT \`user_id\`, COUNT(*) AS \`visits\` + FROM \`ctx1_visits\` + GROUP BY \`user_id\` + HAVING COUNT(*) > 5) + SELECT * FROM \`recurring_users\` + JOIN \`ctx1_users\` ON \`ctx1_users\`.\`id\` = \`recurring_users\`.\`user_id\` + `; + expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) + .toBe(normalizeSQL(expected)); + }); + +// it('should handle multiple CTEs', () => { +// const input = ` +// WITH recurring_users AS ( +// SELECT user_id FROM visits +// ), +// high_value_users AS ( +// SELECT * FROM users +// JOIN recurring_users ON users.id = recurring_users.user_id +// ) +// SELECT * FROM high_value_users +// `; +// const expected = ` +// WITH recurring_users AS ( +// SELECT user_id FROM ctx1_visits +// ), +// high_value_users AS ( +// SELECT * FROM ctx1_users +// JOIN recurring_users ON ctx1_users.id = recurring_users.user_id +// ) +// SELECT * FROM high_value_users +// `; +// expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) +// .toBe(normalizeSQL(expected)); +// }); +// }); + +// describe('Complex Queries - UNIONs', () => { +// it('should handle UNION with subqueries', () => { +// const input = ` +// SELECT * FROM active_users +// WHERE region = 'NA' +// UNION ALL +// SELECT * FROM ( +// SELECT * FROM archived_users +// WHERE deactivation_date > '2023-01-01' +// ) +// UNION ALL +// SELECT * FROM metadata +// `; +// const expected = ` +// SELECT * FROM ctx1_active_users +// WHERE region = 'NA' +// UNION ALL +// SELECT * FROM ( +// SELECT * FROM ctx1_archived_users +// WHERE deactivation_date > '2023-01-01' +// ) +// UNION ALL +// SELECT * FROM metadata +// `; +// expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) +// .toBe(normalizeSQL(expected)); +// }); +// }); + +// describe('Complex Queries - Joins and Subqueries', () => { +// it('should handle complex joins with subqueries', () => { +// const input = ` +// SELECT +// u.name, +// (SELECT COUNT(*) FROM orders o WHERE o.user_id = u.id) as order_count +// FROM users u +// LEFT JOIN profiles p ON u.id = p.user_id +// JOIN ( +// SELECT user_id, COUNT(*) as login_count +// FROM login_history +// GROUP BY user_id +// ) lh ON u.id = lh.user_id +// `; +// const expected = ` +// SELECT +// u.name, +// (SELECT COUNT(*) FROM ctx1_orders o WHERE o.user_id = u.id) as order_count +// FROM ctx1_users u +// LEFT JOIN ctx1_profiles p ON u.id = p.user_id +// JOIN ( +// SELECT user_id, COUNT(*) as login_count +// FROM ctx1_login_history +// GROUP BY user_id +// ) lh ON u.id = lh.user_id +// `; +// expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) +// .toBe(normalizeSQL(expected)); +// }); + +// it('should handle EXISTS subqueries', () => { +// const input = ` +// SELECT * FROM users u +// WHERE EXISTS ( +// SELECT 1 FROM orders +// WHERE orders.user_id = u.id +// ) +// `; +// const expected = ` +// SELECT * FROM ctx1_users u +// WHERE EXISTS ( +// SELECT 1 FROM ctx1_orders +// WHERE ctx1_orders.user_id = u.id +// ) +// `; +// expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) +// .toBe(normalizeSQL(expected)); +// }); +// }); + +// describe('Edge Cases', () => { +// it('should handle empty queries', () => { +// expect(() => transformer.transformQuery('', 'ctx1')).toThrow(); +// }); + +// it('should handle invalid SQL', () => { +// expect(() => transformer.transformQuery('SELECT * FREM users', 'ctx1')).toThrow(); +// }); + +// it('should handle queries with no tables', () => { +// const input = 'SELECT 1 + 1'; +// const expected = 'SELECT 1 + 1'; +// expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) +// .toBe(normalizeSQL(expected)); +// }); + +// it('should handle case sensitivity correctly', () => { +// const input = 'SELECT * FROM Users JOIN METADATA on Users.id = METADATA.user_id'; +// const expected = 'SELECT * FROM ctx1_Users JOIN METADATA on ctx1_Users.id = METADATA.user_id'; +// expect(normalizeSQL(transformer.transformQuery(input, 'ctx1'))) +// .toBe(normalizeSQL(expected)); +// }); + }); +}); \ No newline at end of file diff --git a/src/sql/parser.ts b/src/sql/parser.ts new file mode 100644 index 0000000..60c7fc0 --- /dev/null +++ b/src/sql/parser.ts @@ -0,0 +1,164 @@ +import { Parser } from 'node-sql-parser'; + +export interface TransformationConfig { + excludedTables: Set; + customTransforms?: { + [tableName: string]: (contextId: string) => string; + }; +} + +export class SQLContextTransformer { + private parser: Parser; + private config: TransformationConfig; + private contextId: string = ''; + private cteNames: Set = new Set(); + + constructor(config: TransformationConfig) { + this.parser = new Parser(); + this.config = config; + } + + transformQuery(sql: string, contextId: string): string { + try { + this.contextId = contextId; + this.cteNames.clear(); // Reset CTE names for new query + const ast = this.parser.astify(sql); + + // First pass: collect CTE names + this.collectCTENames(ast); + + // Second pass: transform the AST + const transformedAst = this.transformNode(ast); + + return this.parser.sqlify(transformedAst); + } catch (error) { + console.error('Error in transformQuery:', error); + throw error; + } + } + + private collectCTENames(node: any) { + if (!node) return; + + // Handle WITH clause + if (node.with && Array.isArray(node.with)) { + node.with.forEach((cte: any) => { + if (cte.name) { + this.cteNames.add(cte.name); + } + }); + } + + // Recursively collect from all branches + if (Array.isArray(node)) { + node.forEach(item => this.collectCTENames(item)); + return; + } + + if (typeof node === 'object') { + Object.values(node).forEach(value => this.collectCTENames(value)); + } + } + + private transformNode(node: any): any { + if (!node) return node; + + // Handle arrays + if (Array.isArray(node)) { + return node.map(item => this.transformNode(item)); + } + + // If not an object, return as is + if (typeof node !== 'object') { + return node; + } + + // Clone the node to avoid mutations + let transformedNode = { ...node }; + + // Handle WITH clause + if (transformedNode.with && Array.isArray(transformedNode.with)) { + transformedNode.with = this.transformWithClause(transformedNode.with); + } + + // Handle FROM clause + if (transformedNode.type === 'from') { + transformedNode = this.handleFromClause(transformedNode); + } + + // Handle SELECT statement + if (transformedNode.type === 'select') { + if (transformedNode.from) { + transformedNode.from = this.handleFromClause(transformedNode.from); + } + + // Handle joins + if (transformedNode.join) { + transformedNode.join = transformedNode.join.map((joinClause: any) => ({ + ...joinClause, + table: this.handleFromClause(joinClause.table) + })); + } + } + + // Recursively transform all properties + for (const [key, value] of Object.entries(transformedNode)) { + if (key !== 'from' && key !== 'table' && key !== 'with') { + transformedNode[key] = this.transformNode(value); + } + } + + return transformedNode; + } + + private transformWithClause(withClause: any[]): any[] { + return withClause.map(cte => ({ + ...cte, + name: `${this.contextId}_${cte.name}`, + stmt: this.transformNode(cte.stmt) + })); + } + + private handleFromClause(node: any): any { + if (!node) return node; + + if (Array.isArray(node)) { + return node.map(item => this.handleFromClause(item)); + } + + if (node.table) { + return this.transformTableRef(node); + } + + return node; + } + + private transformTableRef(node: any): any { + const tableName = node.table; + if (!tableName) return node; + + // Clone the node + const transformed = { ...node }; + + // Don't transform excluded tables + if (this.config.excludedTables.has(tableName)) { + return transformed; + } + + // Check if it's a CTE reference + if (this.cteNames.has(tableName)) { + transformed.table = `${this.contextId}_${tableName}`; + return transformed; + } + + // Apply custom transformation if available + if (this.config.customTransforms?.[tableName]) { + transformed.table = this.config.customTransforms[tableName](this.contextId); + return transformed; + } + + // Default transformation + transformed.table = `${this.contextId}_${tableName}`; + return transformed; + } +} \ No newline at end of file diff --git a/src/sqlReparseTables.ts b/src/sqlReparseTables.ts deleted file mode 100644 index 7e35dd9..0000000 --- a/src/sqlReparseTables.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { BaseFrom, From, Parser, Select, With } from "node-sql-parser" -import { generatePrefix } from "./hash" - -const isGlobal = (table: string, globalTables: string[]) => { - return globalTables.map(t => - t.toLowerCase() - ).includes(table.toLowerCase()) -} - -const isBaseFrom = (from: From): from is BaseFrom => { - return Object.keys(from).includes('table') -} - -export const prefixedIfNotGlobal = (tableName: string, globalTables: string[], prefix: string) => { - if (isGlobal(tableName, globalTables)) { - return tableName - } - - return generatePrefix(prefix, tableName) -} - -const updateSelect = (selectAst: Select, globalTables: string[], prefix: string): { selectAst: Select, tables: string[] } => { - - let cteGlobals: string[] = [] - let withs = selectAst.with - let extraTables: string[] = [] - if (selectAst.with) { - cteGlobals = selectAst.with.map(w => w.name.value) - const updatedWiths = selectAst.with.map(w => { - const { selectAst, tables } = updateSelect(w.stmt.ast, [...cteGlobals, ...globalTables], prefix) - extraTables.push(...tables) - return { - ...w, - stmt: { - ...w.stmt, - ast: selectAst - } - } satisfies With - }) - withs = updatedWiths - } - - if (!selectAst.from) { - return { selectAst: selectAst, tables: [] } - } - - const tables: Array = [] - - const updatedFrom = selectAst.from.map(from => { - if(isBaseFrom(from)) { - const t = prefixedIfNotGlobal(from.table, [...cteGlobals, ...globalTables], prefix) - tables.push(t) - return { - ...from, - table: t, - } - } - return from - }) - - return { - selectAst: { - ...selectAst, - from: updatedFrom, - with: withs - }, - tables: [...tables, ...extraTables] - } -} - -export const updateTables = (selectStatement: string, globalTables: string[], prefix: string) => { - const parser = new Parser() - const { ast } = parser.parse(selectStatement) - if (!Array.isArray(ast) && ast.type === 'select') { - const { selectAst, tables } = updateSelect(ast!, globalTables, prefix) - return { statement: parser.sqlify(selectAst), tables } - } else { - throw new Error('Invalid Statement. Only single SELECTs are accepted at the moment.') - } -} - -export const extractCtes = (selectStatement: string) => { - const parser = new Parser() - const { ast } = parser.parse(selectStatement) - if (!Array.isArray(ast) && ast.type === 'select') { - if (ast.with) { - return ast.with.map(w => w.name.value) - } - } - return [] -} \ No newline at end of file diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index 247bc69..6788321 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -2,27 +2,20 @@ import { SqlSealDatabase } from "./database"; import { App } from "obsidian"; import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler"; import { Logger } from "./logger"; -import { TablesManager } from "./dataLoader/collections/tablesManager"; -import { QueryManager } from "./dataLoader/collections/queryManager"; -import { FilesManager } from "./dataLoader/collections/filesManager"; import { RendererRegistry } from "./rendererRegistry"; +import { Sync } from "./datamodel/sync"; export class SqlSeal { public db: SqlSealDatabase public codeBlockHandler: SqlSealCodeblockHandler - public tablesManager: TablesManager constructor(private readonly app: App, verbose = false, rendererRegistry: RendererRegistry) { this.db = new SqlSealDatabase(app, verbose) const logger = new Logger(verbose) - const fileManager = new FilesManager(this.app.vault, this.app, this.db) - this.tablesManager = new TablesManager(fileManager, this.db) - this.tablesManager.getTableSignal('files') - this.tablesManager.getTableSignal('tags') - const queryManager = new QueryManager(this.tablesManager) + const sync = new Sync(this.db, this.app.vault) + sync.init() - this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, logger, this.tablesManager, queryManager, rendererRegistry) - // FIXME: handle here changes to files and tags? + this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, sync, rendererRegistry) } getHandler() { diff --git a/src/utils/signal.test.ts b/src/utils/signal.test.ts deleted file mode 100644 index f5c525e..0000000 --- a/src/utils/signal.test.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { describe, it, expect, jest } from '@jest/globals' -import { createSignal, derivedSignal, withSignals } from './signal'; - -describe('createSignal', () => { - it('should create a signal with initial value', () => { - const signal = createSignal(10); - expect(signal.value).toBe(10); - }); - - it('should create a signal without initial value', () => { - const signal = createSignal(); - expect(signal.value).toBeUndefined(); - }); - - it('should notify subscribers when value changes', () => { - const signal = createSignal(); - const listener = jest.fn(); - - signal(listener); - signal(42); - - expect(listener).toHaveBeenCalledWith(42); - }); - - it('should call listener immediately with current value if exists', () => { - const signal = createSignal(10); - const listener = jest.fn(); - - signal(listener); - expect(listener).toHaveBeenCalledWith(10); - }); - - it('should allow unsubscribing', () => { - const signal = createSignal(); - const listener = jest.fn(); - - const unsubscribe = signal(listener); - signal(42); - expect(listener).toHaveBeenCalledTimes(1); - - unsubscribe(); - signal(43); - expect(listener).toHaveBeenCalledTimes(1); - }); - - it('should allow multiple subscribers', () => { - const signal = createSignal(); - const listener1 = jest.fn(); - const listener2 = jest.fn(); - - signal(listener1); - signal(listener2); - signal(42); - - expect(listener1).toHaveBeenCalledWith(42); - expect(listener2).toHaveBeenCalledWith(42); - }); -}); - -describe('withSignals', () => { - it('should combine multiple signals', () => { - const signal1 = createSignal(); - const signal2 = createSignal(); - const callback = jest.fn(); - - withSignals(signal1, signal2)(callback); - - signal1(42); - expect(callback).not.toHaveBeenCalled(); - - signal2('hello'); - expect(callback).toHaveBeenCalledWith(42, 'hello'); - }); - - it('should work with initial values', () => { - const signal1 = createSignal(42); - const signal2 = createSignal('hello'); - const callback = jest.fn(); - - withSignals(signal1, signal2)(callback); - expect(callback).toHaveBeenCalledWith(42, 'hello'); - }); - - it('should update when any signal changes', () => { - const signal1 = createSignal(42); - const signal2 = createSignal('hello'); - const callback = jest.fn(); - - withSignals(signal1, signal2)(callback); - signal1(43); - - expect(callback).toHaveBeenCalledWith(43, 'hello'); - }); - - it('should allow unsubscribing from all signals', () => { - const signal1 = createSignal(); - const signal2 = createSignal(); - const callback = jest.fn(); - - const unsubscribe = withSignals(signal1, signal2)(callback); - - signal1(42); - signal2('hello'); - expect(callback).toHaveBeenCalledTimes(1); - - unsubscribe(); - signal1(43); - signal2('world'); - expect(callback).toHaveBeenCalledTimes(1); - }); -}); - -describe('derivedSignal', () => { - it('should compute derived value', () => { - const count = createSignal(5); - const multiplier = createSignal(2); - - const product = derivedSignal( - [count, multiplier], - (c, m) => c * m - ); - - expect(product.value).toBe(10); - }); - - it('should update when source signals change', () => { - const count = createSignal(5); - const multiplier = createSignal(2); - - const product = derivedSignal( - [count, multiplier], - (c, m) => c * m - ); - - const listener = jest.fn(); - product(listener); - - count(10); - expect(product.value).toBe(20); - expect(listener).toHaveBeenCalledWith(20); - - multiplier(3); - expect(product.value).toBe(30); - expect(listener).toHaveBeenCalledWith(30); - }); - - it('should handle undefined initial values', () => { - const signal1 = createSignal(); - const signal2 = createSignal(); - - const sum = derivedSignal( - [signal1, signal2], - (a, b) => a + b - ); - - expect(sum.value).toBeUndefined(); - - signal1(5); - expect(sum.value).toBeUndefined(); - - signal2(3); - expect(sum.value).toBe(8); - }); - - it('should work with complex derivations', () => { - const firstName = createSignal('John'); - const lastName = createSignal('Doe'); - const age = createSignal(25); - - const person = derivedSignal( - [firstName, lastName, age], - (first, last, a) => ({ - fullName: `${first} ${last}`, - age: a, - isAdult: a >= 18 - }) - ); - - expect(person.value).toEqual({ - fullName: 'John Doe', - age: 25, - isAdult: true - }); - - firstName('Jane'); - expect(person.value).toEqual({ - fullName: 'Jane Doe', - age: 25, - isAdult: true - }); - }); - - it('should chain derived signals', () => { - const base = createSignal(5); - const doubled = derivedSignal([base], n => n * 2); - const final = derivedSignal([doubled], n => n + 10); - - expect(final.value).toBe(20); - - base(10); - expect(doubled.value).toBe(20); - expect(final.value).toBe(30); - }); - - it('should support array transformations', () => { - const items = createSignal(['apple', 'banana', 'orange']); - const filter = createSignal('an'); - - const filtered = derivedSignal( - [items, filter], - (list, f) => list.filter(item => - item.toLowerCase().includes(f.toLowerCase()) - ) - ); - - expect(filtered.value).toEqual(['banana', 'orange']); - - filter('ap'); - expect(filtered.value).toEqual(['apple']); - - items(['grape', 'apple', 'mango']); - expect(filtered.value).toEqual(['grape', 'apple']); - }); -}); - -describe('type safety', () => { - it('should maintain proper types in derived signals', () => { - const numberSignal = createSignal(42); - const stringSignal = createSignal('hello'); - - // This should type check - const derived = derivedSignal( - [numberSignal, stringSignal], - (num, str) => ({ - number: num, // Should be typed as number - string: str, // Should be typed as string - combined: `${str}${num}` - }) - ); - - expect(derived.value).toEqual({ - number: 42, - string: 'hello', - combined: 'hello42' - }); - }); -}); diff --git a/src/utils/signal.ts b/src/utils/signal.ts deleted file mode 100644 index 644b020..0000000 --- a/src/utils/signal.ts +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Enhanced signal types to include value access - */ -export interface SignalListener { - (event: T): void; -} - -export interface SignalUnsubscriber { - (): void; -} - -export interface SignalSubscriber { - (listener: SignalListener): SignalUnsubscriber; -} - -export interface SignalDispatcher { - (event: T): void; -} - -export interface SignalValue { - readonly value: T; -} - -export type Signal = SignalSubscriber & SignalDispatcher & SignalValue; -export type SignalReturn = SignalUnsubscriber & void; - -export function createSignal(): Signal; -export function createSignal(initialValue: T): Signal; -export function createSignal(initialValue?: T): Signal { - const subscribers = new Set>(); - let currentValue: T | undefined = initialValue; - - const signal = ((eventOrListener: T | SignalListener): any => { - if (typeof eventOrListener === 'function') { - subscribers.add(eventOrListener as SignalListener); - // Call the listener immediately with current value if it exists - if (currentValue !== undefined) { - (eventOrListener as SignalListener)(currentValue); - } - return () => { subscribers.delete(eventOrListener as SignalListener) }; - } else { - currentValue = eventOrListener; - subscribers.forEach(listener => listener(eventOrListener)); - } - }) as Signal; - - // Add value getter - Object.defineProperty(signal, 'value', { - get: () => currentValue, - enumerable: true - }); - - return signal; -} - -/** - * Utility types for handling async compute functions - */ -type UnwrapPromise = T extends Promise ? U : T; - -type ComputeReturnType = - | ((...args: Args) => R) - | ((...args: Args) => Promise); - -/** - * Utility type to extract the event type from a Signal - */ -export type SignalEventType = T extends Signal ? E : never; - -/** - * Creates a new signal that derives its value from other signals - * Supports both synchronous and asynchronous compute functions - */ -export function derivedSignal[], R>( - signals: [...Signals], - compute: ComputeReturnType<{ [K in keyof Signals]: SignalEventType }, R> -): Signal> { - const derivedSignal = createSignal>(); - let computeVersion = 0; - let isComputing = false; - - const updateValue = async (version: number, values: { [K in keyof Signals]: SignalEventType }) => { - if (isComputing) return; - isComputing = true; - - try { - const result = compute(...Object.values(values) as { [K in keyof Signals]: SignalEventType }); - const computedValue = result instanceof Promise ? await result : result; - - // Only update if this is still the latest computation - if (version === computeVersion) { - derivedSignal(computedValue as UnwrapPromise); - } - } catch (error) { - console.error('Error in derived signal computation:', error); - } finally { - isComputing = false; - } - }; - - // Compute initial value if all source signals have values - const initialValues = signals.map(s => s.value); - if (!initialValues.includes(undefined)) { - updateValue(computeVersion, initialValues as { [K in keyof Signals]: SignalEventType }); - } - - // Handle updates - withSignals(...signals)((...args) => { - computeVersion++; - updateValue(computeVersion, args as { [K in keyof Signals]: SignalEventType }); - }); - - return derivedSignal; -} -/** - * Takes multiple signals and returns a function that accepts a callback - * which will receive the values from those signals - */ -export function withSignals[]>( - ...signals: Signals -): ( - callback: (...values: { [K in keyof Signals]: SignalEventType }) => R -) => SignalUnsubscriber { - return (callback) => { - const unsubscribers: SignalUnsubscriber[] = []; - const values = new Array(signals.length) as { [K in keyof Signals]: SignalEventType }; - let initialized = new Array(signals.length).fill(false); - - signals.forEach((signal, index) => { - const unsubscribe = signal((value) => { - values[index] = value as { [K in keyof Signals]: SignalEventType }[number]; - initialized[index] = true; - - if (initialized.every(Boolean)) { - callback(...(values as unknown as { [K in keyof Signals]: SignalEventType })); - } - }); - unsubscribers.push(unsubscribe); - }); - - return () => { - unsubscribers.forEach(unsubscribe => unsubscribe()); - }; - }; -} \ No newline at end of file