From 84f44bfed19c7179de05ed8ab845e245beabd6aa Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sat, 2 Nov 2024 15:23:44 +0000 Subject: [PATCH 1/6] feat: reworking communication to signals and fixing issues with some keys --- jest.config.js | 1 + main.ts | 2 +- package-lock.json | 14 +- src/SealFileSync.ts | 35 ++- src/SqlSealCodeblockHandler.ts | 62 ++++- src/dataLoader/collections/filesManager.ts | 54 +++++ src/dataLoader/collections/queryManager.ts | 21 ++ src/dataLoader/collections/tablesManager.ts | 47 ++++ src/dataLoader/csvFile.ts | 8 + src/dataLoader/dataTransformer.test.ts | 40 ++++ src/dataLoader/dataTransformer.ts | 33 +++ src/dataLoader/tableSignal.ts | 29 +++ src/database.ts | 61 +++-- src/sqlSeal.ts | 11 +- src/utils/sanitiseColumn.ts | 55 +++++ src/utils/signal.test.ts | 247 ++++++++++++++++++++ src/utils/signal.ts | 145 ++++++++++++ yarn.lock | 5 - 18 files changed, 815 insertions(+), 55 deletions(-) create mode 100644 src/dataLoader/collections/filesManager.ts create mode 100644 src/dataLoader/collections/queryManager.ts create mode 100644 src/dataLoader/collections/tablesManager.ts create mode 100644 src/dataLoader/csvFile.ts create mode 100644 src/dataLoader/dataTransformer.test.ts create mode 100644 src/dataLoader/dataTransformer.ts create mode 100644 src/dataLoader/tableSignal.ts create mode 100644 src/utils/sanitiseColumn.ts create mode 100644 src/utils/signal.test.ts create mode 100644 src/utils/signal.ts diff --git a/jest.config.js b/jest.config.js index f5d30d1..6bef2ba 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,5 +3,6 @@ module.exports = { testEnvironment: "node", transform: { "^.+.tsx?$": ["ts-jest",{}], + "^.+.ts?$": ["ts-jest",{}], }, }; \ No newline at end of file diff --git a/main.ts b/main.ts index 22ed3f7..c90c069 100644 --- a/main.ts +++ b/main.ts @@ -18,7 +18,7 @@ export default class SqlSealPlugin extends Plugin { // start syncing when files are loaded this.app.workspace.onLayoutReady(() => { sqlSeal.db.connect().then(() => { - this.fileSync = new SealFileSync(this.app, sqlSeal, this) + this.fileSync = new SealFileSync(this.app, sqlSeal, this, sqlSeal.tablesManager) this.fileSync.init() }) }) diff --git a/package-lock.json b/package-lock.json index ca46513..92bb683 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,14 @@ { "name": "sqlseal", - "version": "0.6.0", + "version": "0.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "sqlseal", - "version": "0.6.0", + "version": "0.8.0", "license": "MIT", "dependencies": { - "antlr4": "^4.13.2", "antlr4ts": "0.5.0-alpha.4", "better-sqlite3": "^11.2.1", "json5": "^2.2.3", @@ -3243,15 +3242,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/antlr4": { - "version": "4.13.2", - "resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz", - "integrity": "sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=16" - } - }, "node_modules/antlr4ts": { "version": "0.5.0-alpha.4", "resolved": "https://registry.npmjs.org/antlr4ts/-/antlr4ts-0.5.0-alpha.4.tgz", diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index e8e994e..2d990bd 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -1,6 +1,8 @@ import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian"; import { SqlSeal } from "./sqlSeal"; import { FieldTypes } from "./utils"; +import { TablesManager } from "./dataLoader/collections/tablesManager"; +import { sanitise } from "./utils/sanitiseColumn"; function fileData(file: TAbstractFile, frontmatter: Record) { return { @@ -12,17 +14,28 @@ function fileData(file: TAbstractFile, frontmatter: Record) { } const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => { - return plugin.app.metadataCache.getFileCache(file)?.frontmatter || {} + const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {} + + return Object.fromEntries( + Object.entries(frontmatter) + .map(([v, s]) => ([sanitise(v), s])) + ) } export class SealFileSync { private currentSchema: Record = {} - constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) { + constructor( + public readonly app: App, + private readonly sqlSeal: SqlSeal, + private readonly plugin: Plugin, + private readonly tableManager: TablesManager + ) { plugin.registerEvent(this.app.vault.on('modify', async (file) => { if (!(file instanceof TFile)) { return } - const frontmatter = extractFrontmatterFromFile(file, plugin) + await sleep(100) + const frontmatter = await extractFrontmatterFromFile(file, plugin) if (this.hasNewColumns(frontmatter)) { await sleep(1000) @@ -35,12 +48,15 @@ export class SealFileSync { await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId') this.sqlSeal.observer.fireObservers('table:files') + this.tableManager.getTableSignal('files')(Date.now()) // Wait 1 second before updating tags table await sleep(1000) await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) this.sqlSeal.observer.fireObservers('table:tags') + this.tableManager.getTableSignal('tags')(Date.now()) + })) plugin.registerEvent(this.app.vault.on('create', async (file) => { @@ -61,11 +77,14 @@ export class SealFileSync { // we need to update the row await this.sqlSeal.db.insertData('files', [fileData(file, frontmatter)]) this.sqlSeal.observer.fireObservers('table:files') + this.tableManager.getTableSignal('files')(Date.now()) // Wait 1 second before updating tags table await sleep(1000) await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) this.sqlSeal.observer.fireObservers('table:tags') + this.tableManager.getTableSignal('tags')(Date.now()) + })) plugin.registerEvent(this.app.vault.on('delete', async (file) => { @@ -75,9 +94,12 @@ export class SealFileSync { await this.sqlSeal.db.deleteData('files', [{ id: file.path }]) this.sqlSeal.observer.fireObservers('table:files') + this.tableManager.getTableSignal('files')(Date.now()) + await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId') this.sqlSeal.observer.fireObservers('table:tags') + this.tableManager.getTableSignal('tags')(Date.now()) })) plugin.registerEvent(this.app.vault.on('rename', async (file, oldPath) => { @@ -93,11 +115,15 @@ export class SealFileSync { await this.sqlSeal.db.insertData('files', [fileData(file, await extractFrontmatterFromFile(file, this.plugin))]) this.sqlSeal.observer.fireObservers('table:files') + this.tableManager.getTableSignal('files')(Date.now()) + // Wait 1 second before updating tags table await sleep(1000) await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) this.sqlSeal.observer.fireObservers('table:tags') + this.tableManager.getTableSignal('tags')(Date.now()) + })) @@ -141,6 +167,9 @@ export class SealFileSync { } this.sqlSeal.observer.fireObservers('table:files') this.sqlSeal.observer.fireObservers('table:tags') + this.tableManager.getTableSignal('files')(Date.now()) + this.tableManager.getTableSignal('tags')(Date.now()) + } hasNewColumns(newFrontmatter: Record) { diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts index 8127de6..e5025b8 100644 --- a/src/SqlSealCodeblockHandler.ts +++ b/src/SqlSealCodeblockHandler.ts @@ -8,6 +8,8 @@ import { SqlSealDatabase } from "./database" import { Logger } from "./logger" import { ParsedLanguage, parseLanguage, TableStatement } from "./grammar/parser" import { SyncModel } from "./models/sync" +import { TablesManager } from "./dataLoader/collections/tablesManager" +import { QueryManager } from "./dataLoader/collections/queryManager" export class SqlSealCodeblockHandler { @@ -15,7 +17,14 @@ export class SqlSealCodeblockHandler { return ['files', 'tags'] } syncModel: SyncModel - constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) { + constructor( + private readonly app: App, + private readonly db: SqlSealDatabase, + private readonly observer: SealObserver, + private logger: Logger, + private tableManager: TablesManager, + private queryManager: QueryManager + ) { this.syncModel = new SyncModel(db) } @@ -66,7 +75,7 @@ export class SqlSealCodeblockHandler { setupSelect(selectStmt: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) { try { - const prefix = hashString(ctx.sourcePath) + const prefix = hashString(ctx.sourcePath) const { statement, tables } = updateTables(selectStmt, this.globalTables, prefix) const frontmatter = resolveFrontmatter(ctx, this.app) @@ -113,22 +122,59 @@ export class SqlSealCodeblockHandler { } } + setupTableSignals(tables: Array) { + tables.forEach(t => { + this.logger.log(`Registering table ${t.name} -> ${t.url}`) + this.tableManager.registerTable(t.name, t.url) + }) + } + + setupQuerySignals({ statement, tables }: ReturnType, el: HTMLElement, ctx: MarkdownPostProcessorContext) { + const frontmatter = resolveFrontmatter(ctx, this.app) + + const renderSelect = async () => { + try { + const stmt = this.db.db.prepare(statement) + const columns = stmt.columns().map(column => column.name); + const data = stmt.all(frontmatter ?? {}) + displayData(el, columns, data, this.app) + } catch (e) { + displayError(el, e) + } + } + + + const sig = this.queryManager.registerQuery(ctx.docId, tables) + sig(() => { + renderSelect() + }) + } + getHandler() { return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => { displayLoader(el) await this.db.connect() - this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers. + // this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers. - // NEW PARSING try { - const results = parseLanguage(source) + const results = parseLanguage(source) - // FIXME: if not properly parsed, we should display error. + const prefix = hashString(ctx.sourcePath) - this.setupTables(results.tables, ctx) - results.queryPart ? this.setupSelect(results.queryPart, el, ctx) : displayInfo(el, `SQLSeal. ${results.tables.map(t => `${t.name} table definition.`).join(' ')}`) + const prefixedTables = results.tables.map(t => { + return { + ...t, + name: prefixedIfNotGlobal(t.name, this.globalTables, prefix) + } + }) + this.setupTableSignals(prefixedTables) + + if (results.queryPart) { + const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix) + this.setupQuerySignals({ statement, tables }, el, ctx) + } } catch (e) { displayError(el, e.toString()) } diff --git a/src/dataLoader/collections/filesManager.ts b/src/dataLoader/collections/filesManager.ts new file mode 100644 index 0000000..5600f1a --- /dev/null +++ b/src/dataLoader/collections/filesManager.ts @@ -0,0 +1,54 @@ +import { Signal, SignalUnsubscriber } from "src/utils/signal"; +import { dataTransformer, DataTransformerOut } from "../dataTransformer"; +import { csvFileSignal } from "../csvFile"; +import { Vault } from "obsidian"; + +export class FilesManager { + files: Map> = new Map() + inputFiles: Map> = new Map() + unregisters: Array = [] + constructor(private vault: Vault) { + this.vault.on('modify', async (file) => { + if (this.files.has(file.path)) { + console.log(`Modified ${file.path}`) + this.inputFiles.get(file.path)!(await this.loadFile(file.path)) + } + }) + } + + private async loadFile(url: string) { + const file = this.vault.getFileByPath(url) + + if (!file) { + return '' + } + const data = await this.vault.cachedRead(file) + return data + } + + destroy() { + this.unregisters.forEach(u => u()) + } + + getFileSignal(filename: string): Signal { + if (this.files.has(filename)) { + return this.files.get(filename)! + } + const fileSignal = csvFileSignal(filename) + const signal = dataTransformer(fileSignal) + + // FIXME: add here ability to watch changes. + + this.files.set(filename, signal) + this.inputFiles.set(filename, fileSignal) + + // now firing signal for the file + this.loadFile(filename).then(fileSignal) + + return signal + } + + addUnregister(unreg: SignalUnsubscriber) { + this.unregisters.push(unreg) + } +} \ No newline at end of file diff --git a/src/dataLoader/collections/queryManager.ts b/src/dataLoader/collections/queryManager.ts new file mode 100644 index 0000000..abbd9c2 --- /dev/null +++ b/src/dataLoader/collections/queryManager.ts @@ -0,0 +1,21 @@ +import { createSignal, derivedSignal, SignalUnsubscriber, withSignals } from "src/utils/signal"; +import { TablesManager } from "./tablesManager"; + +export class QueryManager { + private registeredQueries: Map = new Map() + constructor(private tablesManager: TablesManager) { } + + registerQuery(fileId: string, tables: Array) { + if (this.registeredQueries.has(fileId)) { + this.registeredQueries.get(fileId)!() + this.registeredQueries.delete(fileId) + } + const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t)) + const quertySignal = createSignal() + const unregister = withSignals(...tableSignals)(() => { + quertySignal(Date.now()) + }) + this.registeredQueries.set(fileId, unregister) + return quertySignal + } +} \ No newline at end of file diff --git a/src/dataLoader/collections/tablesManager.ts b/src/dataLoader/collections/tablesManager.ts new file mode 100644 index 0000000..d709a34 --- /dev/null +++ b/src/dataLoader/collections/tablesManager.ts @@ -0,0 +1,47 @@ +import { SqlSealDatabase } from "src/database"; +import { FilesManager } from "./filesManager"; +import { createSignal, Signal, SignalUnsubscriber } from "src/utils/signal"; +import { SignalConstants } from "os"; +import { linkTableWithFile } from "../tableSignal"; + +interface Definition { + fileName: string; + unlink: SignalUnsubscriber; +} + + +export class TablesManager { + private tableLinks: Map = new Map() + private tableSignals: Map> = new Map() + constructor(private filesManager: FilesManager, private db: SqlSealDatabase) { + + } + + registerTable(tableName: string, fileName: string) { + if (this.tableLinks.has(tableName)) { + const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)! + if (prevFileName === fileName) { + console.log('Same filename', fileName) + return + } + console.log('Different Filename') + unlink() + this.tableLinks.delete(tableName) + } + const fileSignal = this.filesManager.getFileSignal(fileName) + const tableSignal = this.getTableSignal(tableName) + + const unlink = linkTableWithFile(fileSignal, tableSignal, tableName, this.db) + this.tableLinks.set(tableName, { + fileName, + unlink + }) + } + + getTableSignal(tableName: string) { + if (!this.tableSignals.has(tableName)) { + this.tableSignals.set(tableName, createSignal()) + } + return this.tableSignals.get(tableName)! + } +} \ No newline at end of file diff --git a/src/dataLoader/csvFile.ts b/src/dataLoader/csvFile.ts new file mode 100644 index 0000000..fa00ed6 --- /dev/null +++ b/src/dataLoader/csvFile.ts @@ -0,0 +1,8 @@ +import { createSignal, Signal } from "src/utils/signal"; + +export type CSVData = string + +export const csvFileSignal = (_file: string): Signal => { + const sig = createSignal() + return sig +} \ No newline at end of file diff --git a/src/dataLoader/dataTransformer.test.ts b/src/dataLoader/dataTransformer.test.ts new file mode 100644 index 0000000..9eb5dc8 --- /dev/null +++ b/src/dataLoader/dataTransformer.test.ts @@ -0,0 +1,40 @@ +import { describe, it, expect, jest } from '@jest/globals' +import { createSignal } from '../utils/signal' +import { dataTransformer } from './dataTransformer' + +describe('Data Transformer', () => { + it('should properly transform basic file', () => { + const s = createSignal() + const dt = dataTransformer(s) + s(`a,b,c +1,2,3 +4,5,6`) + console.log(dt.value) + expect(dt.value.data).toHaveLength(2) + expect(dt.value.data[0]).toEqual({ + a: 1, + b: 2, + c: 3 + }) + + expect(dt.value.types).toEqual({ + a: 'INTEGER', + b: 'INTEGER', + c: 'INTEGER' + }) + }) + + it('should properly transform incorrect keys', () => { + const s = createSignal() + const dt = dataTransformer(s) + s(`BEGIN,PLAN,QUERY,RAISE,beak size (mm) +1.5,hello world,343.423,22.34,2`) + expect(dt.value.types).toEqual({ + 'BEGIN_': 'REAL', + 'PLAN_': 'TEXT', + 'QUERY_': 'REAL', + 'RAISE_': 'REAL', + 'beak_size__mm_': 'INTEGER' + }) + }) +}) \ No newline at end of file diff --git a/src/dataLoader/dataTransformer.ts b/src/dataLoader/dataTransformer.ts new file mode 100644 index 0000000..22c4ad0 --- /dev/null +++ b/src/dataLoader/dataTransformer.ts @@ -0,0 +1,33 @@ +import { derivedSignal, Signal, SignalEventType } from "../utils/signal" +import { CSVData } from "./csvFile" +import { parse } from 'papaparse' +import { toTypeStatements } from "../utils" +import { sanitise } from "../utils/sanitiseColumn" + + +export const dataTransformer = (s: Signal) => { + const sig = derivedSignal([s], (csvData) => { + // FIXME: fix header here. + try { + const parsed = parse>(csvData, { + header: true, + dynamicTyping: false, + skipEmptyLines: true, + transformHeader: sanitise + }) + + const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data) + return typeStatements + } catch (e) { + console.error(e); + return { + data: [], + types: {} + } + } + }) + + return sig +} + +export type DataTransformerOut = SignalEventType> diff --git a/src/dataLoader/tableSignal.ts b/src/dataLoader/tableSignal.ts new file mode 100644 index 0000000..599b979 --- /dev/null +++ b/src/dataLoader/tableSignal.ts @@ -0,0 +1,29 @@ +import { Signal } from "src/utils/signal"; +import { DataTransformerOut } from "./dataTransformer"; +import { SqlSealDatabase } from "src/database"; +import { FieldTypes } from "src/utils"; + +export const linkTableWithFile = (dataSig: Signal, tableSignal: Signal, tableName: string, db: SqlSealDatabase) => { + return dataSig(({ data, types}) => { + + // Check if the columns are exactly the same. If not, delete and reinstantiate the table. + + // For now always remove and reinstantiate the table + db.dropTable(tableName) + + // compute the types of the keys + const columns = Object.entries(types).map(([key, value]) => ({ + name: key, + type: value as FieldTypes + })); + + // create table again + db.createTableClean(tableName, columns) + + // Load all the data + db.insertData(tableName, data) + + // Here we do the actual update of the table. If it succeeds, we return it with current date to indicate sync time. + tableSignal(Date.now()) + }) +} diff --git a/src/database.ts b/src/database.ts index 9ea2b5e..0a26305 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,5 +1,5 @@ import Database from "better-sqlite3" -import { App, normalizePath } from "obsidian" +import { App } from "obsidian" import path from 'path' import Papa from 'papaparse' import { prefixedIfNotGlobal } from "./sqlReparseTables" @@ -7,6 +7,12 @@ import { camelCase } from 'lodash' import { dataToCamelCase, fetchBlobData, FieldTypes, predictJson, predictType, toTypeStatements } from "./utils" import os from 'os' import fs from 'fs' +import { sanitise } from "./utils/sanitiseColumn" + +export interface FieldDefinition { + name: string; + type: FieldTypes +} export class SqlSealDatabase { private savedDatabases: Record = {} @@ -172,38 +178,43 @@ export class SqlSealDatabase { } async insertData(name: string, inData: Array>) { - const data = dataToCamelCase(inData) - const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); - // FIXME: reworking all fields to be camel case - if (!fields || !fields.length) { - return - } - const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); const insertMany = this.db.transaction((pData: Array>) => { + // FIXME: should we do this as a transaction or not? pData.forEach(data => { - try { - // update data so all missing fields are set to null - fields.forEach(field => { - if (typeof data[field] === 'boolean') { - data[field] = data[field] ? 1 : 0 - } else if (!data[field]) { - data[field] = null - } else if (typeof data[field] === 'object' || Array.isArray(data[field])) { - data[field] = JSON.stringify(data[field]) - } - }) - insert.run(data) - } catch (e) { - console.error(e, insert, data) - } + const columns = Object.keys(data) + const insert = this.db.prepare(`INSERT INTO ${name} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`); + const d = Object.keys(data).reduce((ret, key) => { + if (typeof data[key] === 'boolean') { + return data[key] ? 1 : 0; + } + if (!data[key]) { + return null + } + if (typeof data[key] === 'object' || Array.isArray(data[key])) { + return JSON.stringify(data[key]) + } + }, {}) + insert.run(data) }) }) - insertMany(data) + insertMany(inData) + } + + dropTable(name: string) { + this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run() + this.savedDatabases[name] = false + } + + createTableClean(name: string, fields: Array) { + const sqlFields = fields.map(({ name, type }) => `${name} ${type}`).join(', ') + const createSql = `CREATE TABLE IF NOT EXISTS ${name} (${sqlFields})` + this.db.prepare(createSql).run() + this.savedDatabases[name] = true } async createTable(name: string, fields: Record) { - const transformedFiels = Object.entries(fields).map(([key, type]) => [camelCase(key), type]) + const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type]) const uniqueFields = [...new Map(transformedFiels.map(item => [item[0], item])).values()] const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`) diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index a1c2455..c6b4ef8 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -3,17 +3,26 @@ import { App } from "obsidian"; import { SealObserver } from "./SealObserver"; import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler"; import { Logger } from "./logger"; +import { TablesManager } from "./dataLoader/collections/tablesManager"; +import { QueryManager } from "./dataLoader/collections/queryManager"; +import { FilesManager } from "./dataLoader/collections/filesManager"; export class SqlSeal { public db: SqlSealDatabase public observer: SealObserver public codeBlockHandler: SqlSealCodeblockHandler + public tablesManager: TablesManager constructor(private readonly app: App, verbose = false) { this.db = new SqlSealDatabase(app, verbose) this.observer = new SealObserver(verbose) this.observeAllFileChanges() const logger = new Logger(verbose) - this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger) + + const fileManager = new FilesManager(this.app.vault) + this.tablesManager = new TablesManager(fileManager, this.db) + const queryManager = new QueryManager(this.tablesManager) + + this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger, this.tablesManager, queryManager) // FIXME: handle here changes to files and tags? } diff --git a/src/utils/sanitiseColumn.ts b/src/utils/sanitiseColumn.ts new file mode 100644 index 0000000..3fbc03a --- /dev/null +++ b/src/utils/sanitiseColumn.ts @@ -0,0 +1,55 @@ +/** + * Sanitizes a string to be used as a valid SQLite column name. + * Rules implemented: + * 1. Must start with a letter or underscore + * 2. Can only contain letters, numbers, and underscores + * 3. Cannot be a SQLite reserved keyword + * 4. Maximum length is 128 characters (SQLite's practical limit) + * 5. Case-insensitive comparison for reserved keywords + * + * @param input The string to sanitize + * @returns A valid SQLite column name + */ +export function sanitise(input: string): string { + // List of SQLite keywords that cannot be used as column names + const SQLITE_KEYWORDS = new Set([ + 'ABORT', 'ACTION', 'ADD', 'AFTER', 'ALL', 'ALTER', 'ANALYZE', 'AND', 'AS', 'ASC', + 'ATTACH', 'AUTOINCREMENT', 'BEFORE', 'BEGIN', 'BETWEEN', 'BY', 'CASCADE', 'CASE', + 'CAST', 'CHECK', 'COLLATE', 'COLUMN', 'COMMIT', 'CONFLICT', 'CONSTRAINT', 'CREATE', + 'CROSS', 'CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'DATABASE', 'DEFAULT', + 'DEFERRABLE', 'DEFERRED', 'DELETE', 'DESC', 'DETACH', 'DISTINCT', 'DROP', 'EACH', + 'ELSE', 'END', 'ESCAPE', 'EXCEPT', 'EXCLUSIVE', 'EXISTS', 'EXPLAIN', 'FAIL', + 'FOR', 'FOREIGN', 'FROM', 'FULL', 'GLOB', 'GROUP', 'HAVING', 'IF', 'IGNORE', + 'IMMEDIATE', 'IN', 'INDEX', 'INDEXED', 'INITIALLY', 'INNER', 'INSERT', 'INSTEAD', + 'INTERSECT', 'INTO', 'IS', 'ISNULL', 'JOIN', 'KEY', 'LEFT', 'LIKE', 'LIMIT', + 'MATCH', 'NATURAL', 'NO', 'NOT', 'NOTNULL', 'NULL', 'OF', 'OFFSET', 'ON', + 'OR', 'ORDER', 'OUTER', 'PLAN', 'PRAGMA', 'PRIMARY', 'QUERY', 'RAISE', + 'RECURSIVE', 'REFERENCES', 'REGEXP', 'REINDEX', 'RELEASE', 'RENAME', 'REPLACE', + 'RESTRICT', 'RIGHT', 'ROLLBACK', 'ROW', 'SAVEPOINT', 'SELECT', 'SET', + 'TABLE', 'TEMP', 'TEMPORARY', 'THEN', 'TO', 'TRANSACTION', 'TRIGGER', 'UNION', + 'UNIQUE', 'UPDATE', 'USING', 'VACUUM', 'VALUES', 'VIEW', 'VIRTUAL', 'WHEN', + 'WHERE', 'WITH', 'WITHOUT' + ]); + + if (!input || typeof input !== 'string') { + return '_empty'; + } + + // Replace any character that isn't a letter, number, or underscore with underscore + let sanitized = input.replace(/[^a-zA-Z0-9_]/g, '_'); + + // Ensure it starts with a letter or underscore + if (!/^[a-zA-Z_]/.test(sanitized)) { + sanitized = '_' + sanitized; + } + + // Truncate to maximum length + sanitized = sanitized.slice(0, 128); + + // If it's a reserved keyword, append underscore + if (SQLITE_KEYWORDS.has(sanitized.toUpperCase())) { + sanitized += '_'; + } + + return sanitized; +} \ No newline at end of file diff --git a/src/utils/signal.test.ts b/src/utils/signal.test.ts new file mode 100644 index 0000000..f5c525e --- /dev/null +++ b/src/utils/signal.test.ts @@ -0,0 +1,247 @@ +import { describe, it, expect, jest } from '@jest/globals' +import { createSignal, derivedSignal, withSignals } from './signal'; + +describe('createSignal', () => { + it('should create a signal with initial value', () => { + const signal = createSignal(10); + expect(signal.value).toBe(10); + }); + + it('should create a signal without initial value', () => { + const signal = createSignal(); + expect(signal.value).toBeUndefined(); + }); + + it('should notify subscribers when value changes', () => { + const signal = createSignal(); + const listener = jest.fn(); + + signal(listener); + signal(42); + + expect(listener).toHaveBeenCalledWith(42); + }); + + it('should call listener immediately with current value if exists', () => { + const signal = createSignal(10); + const listener = jest.fn(); + + signal(listener); + expect(listener).toHaveBeenCalledWith(10); + }); + + it('should allow unsubscribing', () => { + const signal = createSignal(); + const listener = jest.fn(); + + const unsubscribe = signal(listener); + signal(42); + expect(listener).toHaveBeenCalledTimes(1); + + unsubscribe(); + signal(43); + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('should allow multiple subscribers', () => { + const signal = createSignal(); + const listener1 = jest.fn(); + const listener2 = jest.fn(); + + signal(listener1); + signal(listener2); + signal(42); + + expect(listener1).toHaveBeenCalledWith(42); + expect(listener2).toHaveBeenCalledWith(42); + }); +}); + +describe('withSignals', () => { + it('should combine multiple signals', () => { + const signal1 = createSignal(); + const signal2 = createSignal(); + const callback = jest.fn(); + + withSignals(signal1, signal2)(callback); + + signal1(42); + expect(callback).not.toHaveBeenCalled(); + + signal2('hello'); + expect(callback).toHaveBeenCalledWith(42, 'hello'); + }); + + it('should work with initial values', () => { + const signal1 = createSignal(42); + const signal2 = createSignal('hello'); + const callback = jest.fn(); + + withSignals(signal1, signal2)(callback); + expect(callback).toHaveBeenCalledWith(42, 'hello'); + }); + + it('should update when any signal changes', () => { + const signal1 = createSignal(42); + const signal2 = createSignal('hello'); + const callback = jest.fn(); + + withSignals(signal1, signal2)(callback); + signal1(43); + + expect(callback).toHaveBeenCalledWith(43, 'hello'); + }); + + it('should allow unsubscribing from all signals', () => { + const signal1 = createSignal(); + const signal2 = createSignal(); + const callback = jest.fn(); + + const unsubscribe = withSignals(signal1, signal2)(callback); + + signal1(42); + signal2('hello'); + expect(callback).toHaveBeenCalledTimes(1); + + unsubscribe(); + signal1(43); + signal2('world'); + expect(callback).toHaveBeenCalledTimes(1); + }); +}); + +describe('derivedSignal', () => { + it('should compute derived value', () => { + const count = createSignal(5); + const multiplier = createSignal(2); + + const product = derivedSignal( + [count, multiplier], + (c, m) => c * m + ); + + expect(product.value).toBe(10); + }); + + it('should update when source signals change', () => { + const count = createSignal(5); + const multiplier = createSignal(2); + + const product = derivedSignal( + [count, multiplier], + (c, m) => c * m + ); + + const listener = jest.fn(); + product(listener); + + count(10); + expect(product.value).toBe(20); + expect(listener).toHaveBeenCalledWith(20); + + multiplier(3); + expect(product.value).toBe(30); + expect(listener).toHaveBeenCalledWith(30); + }); + + it('should handle undefined initial values', () => { + const signal1 = createSignal(); + const signal2 = createSignal(); + + const sum = derivedSignal( + [signal1, signal2], + (a, b) => a + b + ); + + expect(sum.value).toBeUndefined(); + + signal1(5); + expect(sum.value).toBeUndefined(); + + signal2(3); + expect(sum.value).toBe(8); + }); + + it('should work with complex derivations', () => { + const firstName = createSignal('John'); + const lastName = createSignal('Doe'); + const age = createSignal(25); + + const person = derivedSignal( + [firstName, lastName, age], + (first, last, a) => ({ + fullName: `${first} ${last}`, + age: a, + isAdult: a >= 18 + }) + ); + + expect(person.value).toEqual({ + fullName: 'John Doe', + age: 25, + isAdult: true + }); + + firstName('Jane'); + expect(person.value).toEqual({ + fullName: 'Jane Doe', + age: 25, + isAdult: true + }); + }); + + it('should chain derived signals', () => { + const base = createSignal(5); + const doubled = derivedSignal([base], n => n * 2); + const final = derivedSignal([doubled], n => n + 10); + + expect(final.value).toBe(20); + + base(10); + expect(doubled.value).toBe(20); + expect(final.value).toBe(30); + }); + + it('should support array transformations', () => { + const items = createSignal(['apple', 'banana', 'orange']); + const filter = createSignal('an'); + + const filtered = derivedSignal( + [items, filter], + (list, f) => list.filter(item => + item.toLowerCase().includes(f.toLowerCase()) + ) + ); + + expect(filtered.value).toEqual(['banana', 'orange']); + + filter('ap'); + expect(filtered.value).toEqual(['apple']); + + items(['grape', 'apple', 'mango']); + expect(filtered.value).toEqual(['grape', 'apple']); + }); +}); + +describe('type safety', () => { + it('should maintain proper types in derived signals', () => { + const numberSignal = createSignal(42); + const stringSignal = createSignal('hello'); + + // This should type check + const derived = derivedSignal( + [numberSignal, stringSignal], + (num, str) => ({ + number: num, // Should be typed as number + string: str, // Should be typed as string + combined: `${str}${num}` + }) + ); + + expect(derived.value).toEqual({ + number: 42, + string: 'hello', + combined: 'hello42' + }); + }); +}); diff --git a/src/utils/signal.ts b/src/utils/signal.ts new file mode 100644 index 0000000..644b020 --- /dev/null +++ b/src/utils/signal.ts @@ -0,0 +1,145 @@ +/** + * Enhanced signal types to include value access + */ +export interface SignalListener { + (event: T): void; +} + +export interface SignalUnsubscriber { + (): void; +} + +export interface SignalSubscriber { + (listener: SignalListener): SignalUnsubscriber; +} + +export interface SignalDispatcher { + (event: T): void; +} + +export interface SignalValue { + readonly value: T; +} + +export type Signal = SignalSubscriber & SignalDispatcher & SignalValue; +export type SignalReturn = SignalUnsubscriber & void; + +export function createSignal(): Signal; +export function createSignal(initialValue: T): Signal; +export function createSignal(initialValue?: T): Signal { + const subscribers = new Set>(); + let currentValue: T | undefined = initialValue; + + const signal = ((eventOrListener: T | SignalListener): any => { + if (typeof eventOrListener === 'function') { + subscribers.add(eventOrListener as SignalListener); + // Call the listener immediately with current value if it exists + if (currentValue !== undefined) { + (eventOrListener as SignalListener)(currentValue); + } + return () => { subscribers.delete(eventOrListener as SignalListener) }; + } else { + currentValue = eventOrListener; + subscribers.forEach(listener => listener(eventOrListener)); + } + }) as Signal; + + // Add value getter + Object.defineProperty(signal, 'value', { + get: () => currentValue, + enumerable: true + }); + + return signal; +} + +/** + * Utility types for handling async compute functions + */ +type UnwrapPromise = T extends Promise ? U : T; + +type ComputeReturnType = + | ((...args: Args) => R) + | ((...args: Args) => Promise); + +/** + * Utility type to extract the event type from a Signal + */ +export type SignalEventType = T extends Signal ? E : never; + +/** + * Creates a new signal that derives its value from other signals + * Supports both synchronous and asynchronous compute functions + */ +export function derivedSignal[], R>( + signals: [...Signals], + compute: ComputeReturnType<{ [K in keyof Signals]: SignalEventType }, R> +): Signal> { + const derivedSignal = createSignal>(); + let computeVersion = 0; + let isComputing = false; + + const updateValue = async (version: number, values: { [K in keyof Signals]: SignalEventType }) => { + if (isComputing) return; + isComputing = true; + + try { + const result = compute(...Object.values(values) as { [K in keyof Signals]: SignalEventType }); + const computedValue = result instanceof Promise ? await result : result; + + // Only update if this is still the latest computation + if (version === computeVersion) { + derivedSignal(computedValue as UnwrapPromise); + } + } catch (error) { + console.error('Error in derived signal computation:', error); + } finally { + isComputing = false; + } + }; + + // Compute initial value if all source signals have values + const initialValues = signals.map(s => s.value); + if (!initialValues.includes(undefined)) { + updateValue(computeVersion, initialValues as { [K in keyof Signals]: SignalEventType }); + } + + // Handle updates + withSignals(...signals)((...args) => { + computeVersion++; + updateValue(computeVersion, args as { [K in keyof Signals]: SignalEventType }); + }); + + return derivedSignal; +} +/** + * Takes multiple signals and returns a function that accepts a callback + * which will receive the values from those signals + */ +export function withSignals[]>( + ...signals: Signals +): ( + callback: (...values: { [K in keyof Signals]: SignalEventType }) => R +) => SignalUnsubscriber { + return (callback) => { + const unsubscribers: SignalUnsubscriber[] = []; + const values = new Array(signals.length) as { [K in keyof Signals]: SignalEventType }; + let initialized = new Array(signals.length).fill(false); + + signals.forEach((signal, index) => { + const unsubscribe = signal((value) => { + values[index] = value as { [K in keyof Signals]: SignalEventType }[number]; + initialized[index] = true; + + if (initialized.every(Boolean)) { + callback(...(values as unknown as { [K in keyof Signals]: SignalEventType })); + } + }); + unsubscribers.push(unsubscribe); + }); + + return () => { + unsubscribers.forEach(unsubscribe => unsubscribe()); + }; + }; +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index a9f6fc7..c98473c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1428,11 +1428,6 @@ ansi-styles@^5.0.0: resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== -antlr4@^4.13.2: - version "4.13.2" - resolved "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz" - integrity sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg== - antlr4ts-cli@0.5.0-alpha.4: version "0.5.0-alpha.4" resolved "https://registry.npmjs.org/antlr4ts-cli/-/antlr4ts-cli-0.5.0-alpha.4.tgz" From f34f0476f2bf951ffc7d9a2a1858889d015eddac Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 3 Nov 2024 10:05:41 +0000 Subject: [PATCH 2/6] chore: fixing issue with tags --- main.ts | 2 +- src/SealFileSync.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/main.ts b/main.ts index c90c069..81f5c42 100644 --- a/main.ts +++ b/main.ts @@ -12,7 +12,7 @@ export default class SqlSealPlugin extends Plugin { async onload() { await this.loadSettings(); - const sqlSeal = new SqlSeal(this.app, false) // FIXME: set verbose based on the env. + const sqlSeal = new SqlSeal(this.app, true) // FIXME: set verbose based on the env. this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) this.sqlSeal = sqlSeal // start syncing when files are loaded diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 2d990bd..9c984c9 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -7,6 +7,7 @@ import { sanitise } from "./utils/sanitiseColumn"; function fileData(file: TAbstractFile, frontmatter: Record) { return { ...frontmatter, + tags: undefined, path: file.path, name: file.name.replace(/\.[^/.]+$/, ""), id: file.path @@ -138,9 +139,13 @@ export class SealFileSync { } async getFileTags(file: TFile) { - return (this.app.metadataCache.getFileCache(file)?.tags || []) - .map(t => t.tag) + console.log('file', file.basename) + console.log(this.app.metadataCache.getFileCache(file)) + const t = ((this.app.metadataCache.getFileCache(file)?.frontmatter?.tags || []) as string[]) + // .map(t => t.tag) .map(t => ({ tag: t, fileId: file.path })) + console.log('FILE TAGS', t) + return t } async init() { From 6fbcbd2157e4926696ecb3c618406f161fef5d25 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 3 Nov 2024 10:56:33 +0000 Subject: [PATCH 3/6] chore: removing old event based code --- src/SealFileSync.ts | 10 --- src/SealObserver.ts | 77 ---------------- src/SqlSealCodeblockHandler.ts | 99 +-------------------- src/dataLoader/collections/queryManager.ts | 2 +- src/dataLoader/collections/tablesManager.ts | 5 +- src/sqlSeal.ts | 13 +-- 6 files changed, 7 insertions(+), 199 deletions(-) delete mode 100644 src/SealObserver.ts diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 9c984c9..162de37 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -48,14 +48,12 @@ export class SealFileSync { await this.sqlSeal.db.updateData('files', [fileData(file, frontmatter)]) await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId') - this.sqlSeal.observer.fireObservers('table:files') this.tableManager.getTableSignal('files')(Date.now()) // Wait 1 second before updating tags table await sleep(1000) await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) - this.sqlSeal.observer.fireObservers('table:tags') this.tableManager.getTableSignal('tags')(Date.now()) })) @@ -77,13 +75,11 @@ export class SealFileSync { // we need to update the row await this.sqlSeal.db.insertData('files', [fileData(file, frontmatter)]) - this.sqlSeal.observer.fireObservers('table:files') this.tableManager.getTableSignal('files')(Date.now()) // Wait 1 second before updating tags table await sleep(1000) await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) - this.sqlSeal.observer.fireObservers('table:tags') this.tableManager.getTableSignal('tags')(Date.now()) })) @@ -94,12 +90,10 @@ export class SealFileSync { } await this.sqlSeal.db.deleteData('files', [{ id: file.path }]) - this.sqlSeal.observer.fireObservers('table:files') this.tableManager.getTableSignal('files')(Date.now()) await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId') - this.sqlSeal.observer.fireObservers('table:tags') this.tableManager.getTableSignal('tags')(Date.now()) })) @@ -115,14 +109,12 @@ export class SealFileSync { await this.sqlSeal.db.deleteData('tags', [{ fileId: oldPath }], 'fileId') await this.sqlSeal.db.insertData('files', [fileData(file, await extractFrontmatterFromFile(file, this.plugin))]) - this.sqlSeal.observer.fireObservers('table:files') this.tableManager.getTableSignal('files')(Date.now()) // Wait 1 second before updating tags table await sleep(1000) await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) - this.sqlSeal.observer.fireObservers('table:tags') this.tableManager.getTableSignal('tags')(Date.now()) @@ -170,8 +162,6 @@ export class SealFileSync { 'fileId': 'TEXT' }) } - this.sqlSeal.observer.fireObservers('table:files') - this.sqlSeal.observer.fireObservers('table:tags') this.tableManager.getTableSignal('files')(Date.now()) this.tableManager.getTableSignal('tags')(Date.now()) diff --git a/src/SealObserver.ts b/src/SealObserver.ts deleted file mode 100644 index 2f3aaa6..0000000 --- a/src/SealObserver.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Logger } from "./logger"; - -type Callback = () => void; - -export class SealObserver { - private tables: Map>; - - private tags: Map>; - private logger: Logger - - constructor(verbose = false) { - this.tables = new Map(); - this.tags = new Map(); - this.logger = new Logger(verbose) - } - - registerObserver(tableNames: string | string[], observer: Callback, tag?: string) { - if (typeof tableNames === 'string') { - tableNames = [tableNames]; - } - tableNames.forEach(tableName => { - if (!this.tables.has(tableName)) { - this.tables.set(tableName, new Set()); - this.logger.log(`Table "${tableName}" registered.`); - } - const observers = this.tables.get(tableName); - if (observers !== undefined) { - observers.add(observer); - this.logger.log(`Observer registered for "${tableName}".`); - } - if (tag) { - if (!this.tags.has(tag)) { - this.tags.set(tag, new Set()); - } - const tags = this.tags.get(tag); - if (tags !== undefined) { - tags.add(observer); - } - } - }); - } - - hasAnyObserver(tableNames: string[]) { - return tableNames.reduce((acc, t) => - acc || (this.tables.has(t) && this.tables.get(t)!.size > 0), - false) - } - - unregisterObserversByTag(tag: string) { - if (this.tags.has(tag)) { - const observers = this.tags.get(tag); - if (observers) { - observers.forEach(observer => { - this.unregisterObserver(observer) - }) - this.tags.delete(tag) - } - } - } - - unregisterObserver(observer: Callback) { - this.tables.forEach(observers => { - observers.delete(observer); - }); - this.logger.log(`Observer unregistered.`); - } - - fireObservers(tableName: string) { - if (this.tables.has(tableName)) { - const observers = this.tables.get(tableName); - if (observers) { - observers.forEach(observer => observer()); - this.logger.log(`Observers fired for table "${tableName}".`); - } - } - } -} \ No newline at end of file diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts index e5025b8..8589c2a 100644 --- a/src/SqlSealCodeblockHandler.ts +++ b/src/SqlSealCodeblockHandler.ts @@ -3,10 +3,9 @@ import { displayData, displayError, displayInfo, displayLoader } from "./ui" import { resolveFrontmatter } from "./frontmatter" import { hashString } from "./hash" import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables" -import { SealObserver } from "./SealObserver" import { SqlSealDatabase } from "./database" import { Logger } from "./logger" -import { ParsedLanguage, parseLanguage, TableStatement } from "./grammar/parser" +import { parseLanguage, TableStatement } from "./grammar/parser" import { SyncModel } from "./models/sync" import { TablesManager } from "./dataLoader/collections/tablesManager" import { QueryManager } from "./dataLoader/collections/queryManager" @@ -20,7 +19,6 @@ export class SqlSealCodeblockHandler { constructor( private readonly app: App, private readonly db: SqlSealDatabase, - private readonly observer: SealObserver, private logger: Logger, private tableManager: TablesManager, private queryManager: QueryManager @@ -28,100 +26,6 @@ export class SqlSealCodeblockHandler { this.syncModel = new SyncModel(db) } - setupTables(tables: ParsedLanguage['tables'], ctx: MarkdownPostProcessorContext) { - tables.forEach(table => { - this.setupTable(table, ctx) - }) - } - - setupTable({ url, name }: TableStatement, ctx: MarkdownPostProcessorContext) { - // UPDATING TABLES IF THEY CHANGED SINCE LAST REGISTER - const prefix = hashString(ctx.sourcePath) - const prefixedName = prefixedIfNotGlobal(name, this.globalTables, prefix) - - const existing = this.syncModel.getSync(ctx.sourcePath, prefixedName) - - this.observer.registerObserver(`file:${url}`, async () => { - // Update table - await this.db.loadDataForDatabaseFromUrl(prefixedName, url, true) - - // update when it was updated. - this.syncModel.removeSync(ctx.sourcePath, prefixedName) - this.syncModel.registerSync(ctx.sourcePath, url, prefixedName) - - // Fire observers for the table - this.observer.fireObservers(`table:${prefixedName}`) - }, ctx.docId) - - if (existing) { - - if (existing.url === url) { - // Already synced - // TODO: Check if the file was updated after we synced, if so, we should update it. - requestAnimationFrame(() => { - this.observer.fireObservers(`table:${prefixedName}`) - }) - return - } else { - // Old database is no-no. We need to update it. - this.db.db.prepare('DROP TABLE :tablename').run({ - tablename: prefixedName - }) - this.syncModel.removeSync(ctx.sourcePath, prefixedName) - } - } - this.observer.fireObservers(`file:${url}`) - } - - setupSelect(selectStmt: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) { - try { - const prefix = hashString(ctx.sourcePath) - const { statement, tables } = updateTables(selectStmt, this.globalTables, prefix) - const frontmatter = resolveFrontmatter(ctx, this.app) - - const renderSelect = async () => { - try { - const stmt = this.db.db.prepare(statement) - const columns = stmt.columns().map(column => column.name); - const data = stmt.all(frontmatter ?? {}) - displayData(el, columns, data, this.app) - } catch (e) { - displayError(el, e) - } - } - // Register observer for each table - tables.forEach(table => { - const observer = async () => { - if (!el.isConnected) { - // Unregistering using the context id - this.observer.unregisterObserversByTag(ctx.docId) - } - - displayLoader(el) - - await renderSelect() - } - this.observer.registerObserver(`table:${table}`, observer, ctx.docId) - }) - this.globalTables.forEach(t => this.observer.fireObservers(`table:${t}`)) - if (!this.observer.hasAnyObserver(this.globalTables.map(t => `table:${t}`))) { - renderSelect() - } - // Triggering the ones which exist - tables.forEach(t => { - if (this.syncModel.getSync(ctx.sourcePath, t)) { - this.observer.fireObservers(`table:${t}`) - } - }) - } catch (e) { - if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) { - displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.') - } else { - displayError(el, e) - } - } - } - setupTableSignals(tables: Array) { tables.forEach(t => { this.logger.log(`Registering table ${t.name} -> ${t.url}`) @@ -155,7 +59,6 @@ export class SqlSealCodeblockHandler { displayLoader(el) await this.db.connect() - // this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers. try { const results = parseLanguage(source) diff --git a/src/dataLoader/collections/queryManager.ts b/src/dataLoader/collections/queryManager.ts index abbd9c2..1b09553 100644 --- a/src/dataLoader/collections/queryManager.ts +++ b/src/dataLoader/collections/queryManager.ts @@ -10,7 +10,7 @@ export class QueryManager { this.registeredQueries.get(fileId)!() this.registeredQueries.delete(fileId) } - const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t)) + const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t, true)) const quertySignal = createSignal() const unregister = withSignals(...tableSignals)(() => { quertySignal(Date.now()) diff --git a/src/dataLoader/collections/tablesManager.ts b/src/dataLoader/collections/tablesManager.ts index d709a34..3c7c338 100644 --- a/src/dataLoader/collections/tablesManager.ts +++ b/src/dataLoader/collections/tablesManager.ts @@ -38,8 +38,11 @@ export class TablesManager { }) } - getTableSignal(tableName: string) { + getTableSignal(tableName: string, failOnUndefined: boolean = false) { if (!this.tableSignals.has(tableName)) { + if (failOnUndefined) { + throw new Error(`${tableName} does not exist`) + } this.tableSignals.set(tableName, createSignal()) } return this.tableSignals.get(tableName)! diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index c6b4ef8..bb6b1ed 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -1,6 +1,5 @@ import { SqlSealDatabase } from "./database"; import { App } from "obsidian"; -import { SealObserver } from "./SealObserver"; import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler"; import { Logger } from "./logger"; import { TablesManager } from "./dataLoader/collections/tablesManager"; @@ -9,31 +8,21 @@ import { FilesManager } from "./dataLoader/collections/filesManager"; export class SqlSeal { public db: SqlSealDatabase - public observer: SealObserver public codeBlockHandler: SqlSealCodeblockHandler public tablesManager: TablesManager constructor(private readonly app: App, verbose = false) { this.db = new SqlSealDatabase(app, verbose) - this.observer = new SealObserver(verbose) - this.observeAllFileChanges() const logger = new Logger(verbose) const fileManager = new FilesManager(this.app.vault) this.tablesManager = new TablesManager(fileManager, this.db) const queryManager = new QueryManager(this.tablesManager) - this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger, this.tablesManager, queryManager) + this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, logger, this.tablesManager, queryManager) // FIXME: handle here changes to files and tags? } getHandler() { return this.codeBlockHandler.getHandler() } - - private observeAllFileChanges() { - // Use fs to observe file changes - this.app.vault.on('modify', async (file) => { - this.observer.fireObservers('file:' + file.path) - }) - } } From 182ee4bc8bef80f6ca96110e8a07f2e2cb71dc5d Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 3 Nov 2024 11:00:14 +0000 Subject: [PATCH 4/6] chore: remove console logs --- src/SealFileSync.ts | 4 ---- src/dataLoader/collections/filesManager.ts | 1 - src/dataLoader/collections/tablesManager.ts | 2 -- src/dataLoader/dataTransformer.test.ts | 1 - 4 files changed, 8 deletions(-) diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 162de37..6053bf1 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -131,12 +131,8 @@ export class SealFileSync { } async getFileTags(file: TFile) { - console.log('file', file.basename) - console.log(this.app.metadataCache.getFileCache(file)) const t = ((this.app.metadataCache.getFileCache(file)?.frontmatter?.tags || []) as string[]) - // .map(t => t.tag) .map(t => ({ tag: t, fileId: file.path })) - console.log('FILE TAGS', t) return t } diff --git a/src/dataLoader/collections/filesManager.ts b/src/dataLoader/collections/filesManager.ts index 5600f1a..25b0ec5 100644 --- a/src/dataLoader/collections/filesManager.ts +++ b/src/dataLoader/collections/filesManager.ts @@ -10,7 +10,6 @@ export class FilesManager { constructor(private vault: Vault) { this.vault.on('modify', async (file) => { if (this.files.has(file.path)) { - console.log(`Modified ${file.path}`) this.inputFiles.get(file.path)!(await this.loadFile(file.path)) } }) diff --git a/src/dataLoader/collections/tablesManager.ts b/src/dataLoader/collections/tablesManager.ts index 3c7c338..b03ce13 100644 --- a/src/dataLoader/collections/tablesManager.ts +++ b/src/dataLoader/collections/tablesManager.ts @@ -21,10 +21,8 @@ export class TablesManager { if (this.tableLinks.has(tableName)) { const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)! if (prevFileName === fileName) { - console.log('Same filename', fileName) return } - console.log('Different Filename') unlink() this.tableLinks.delete(tableName) } diff --git a/src/dataLoader/dataTransformer.test.ts b/src/dataLoader/dataTransformer.test.ts index 9eb5dc8..3d6beee 100644 --- a/src/dataLoader/dataTransformer.test.ts +++ b/src/dataLoader/dataTransformer.test.ts @@ -9,7 +9,6 @@ describe('Data Transformer', () => { s(`a,b,c 1,2,3 4,5,6`) - console.log(dt.value) expect(dt.value.data).toHaveLength(2) expect(dt.value.data[0]).toEqual({ a: 1, From 251bdb42065f768c0e30349dc36c85bc795d788c Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 3 Nov 2024 11:24:36 +0000 Subject: [PATCH 5/6] chore: converting column names to lower-case. Updating how data formatting works --- src/dataLoader/dataTransformer.test.ts | 8 ++++---- src/database.ts | 21 +++++++++++++++++---- src/utils/sanitiseColumn.ts | 2 +- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/dataLoader/dataTransformer.test.ts b/src/dataLoader/dataTransformer.test.ts index 3d6beee..ff967d9 100644 --- a/src/dataLoader/dataTransformer.test.ts +++ b/src/dataLoader/dataTransformer.test.ts @@ -29,10 +29,10 @@ describe('Data Transformer', () => { s(`BEGIN,PLAN,QUERY,RAISE,beak size (mm) 1.5,hello world,343.423,22.34,2`) expect(dt.value.types).toEqual({ - 'BEGIN_': 'REAL', - 'PLAN_': 'TEXT', - 'QUERY_': 'REAL', - 'RAISE_': 'REAL', + 'begin_': 'REAL', + 'plan_': 'TEXT', + 'query_': 'REAL', + 'raise_': 'REAL', 'beak_size__mm_': 'INTEGER' }) }) diff --git a/src/database.ts b/src/database.ts index 0a26305..dd1e213 100644 --- a/src/database.ts +++ b/src/database.ts @@ -185,16 +185,29 @@ export class SqlSealDatabase { const insert = this.db.prepare(`INSERT INTO ${name} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`); const d = Object.keys(data).reduce((ret, key) => { if (typeof data[key] === 'boolean') { - return data[key] ? 1 : 0; + return { + ...ret, + [key]: data[key] ? 1 : 0 + } } if (!data[key]) { - return null + return { + ...ret, + [key]: null + } } if (typeof data[key] === 'object' || Array.isArray(data[key])) { - return JSON.stringify(data[key]) + return { + ...ret, + [key]: JSON.stringify(data[key]) + } + } + return { + ...ret, + [key]: data[key] } }, {}) - insert.run(data) + insert.run(d) }) }) diff --git a/src/utils/sanitiseColumn.ts b/src/utils/sanitiseColumn.ts index 3fbc03a..e1da8ae 100644 --- a/src/utils/sanitiseColumn.ts +++ b/src/utils/sanitiseColumn.ts @@ -51,5 +51,5 @@ export function sanitise(input: string): string { sanitized += '_'; } - return sanitized; + return sanitized.toLowerCase(); } \ No newline at end of file From bd136d5d17f83175b351912849c50a3abc93b078 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 3 Nov 2024 20:43:13 +0000 Subject: [PATCH 6/6] feat: fixing issue with tags loading --- src/SealFileSync.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 6053bf1..ffcbbd4 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -1,13 +1,12 @@ -import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian"; +import { App, EventRef, getAllTags, Plugin, TAbstractFile, TFile } from "obsidian"; import { SqlSeal } from "./sqlSeal"; import { FieldTypes } from "./utils"; import { TablesManager } from "./dataLoader/collections/tablesManager"; import { sanitise } from "./utils/sanitiseColumn"; -function fileData(file: TAbstractFile, frontmatter: Record) { +function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record) { return { ...frontmatter, - tags: undefined, path: file.path, name: file.name.replace(/\.[^/.]+$/, ""), id: file.path @@ -131,9 +130,18 @@ export class SealFileSync { } async getFileTags(file: TFile) { - const t = ((this.app.metadataCache.getFileCache(file)?.frontmatter?.tags || []) as string[]) - .map(t => ({ tag: t, fileId: file.path })) - return t + const cache = this.app.metadataCache.getFileCache(file) + if (!cache) { + return [] + } + const tags = getAllTags(cache) + if (!tags) { + return [] + } + return tags.map((t) => ({ + tag: t, + fileId: file.path + })) } async init() {