diff --git a/CHANGELOG.md b/CHANGELOG.md index 7daa252..2f8410a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.18.0 (2025-01-23) +Synchronisation code has been greatly refactored. Thanks to that many common issue with files not being refreshed or pointing to the wrong file occasionally should be fixed. +Also fixed many instances of potential memory leaks. Now plugin should be more stable and reliable. + # 0.17.0 (2025-01-20) Adding support for `LIST` view. You can now set custom classes for `HTML` (table) view. diff --git a/manifest.json b/manifest.json index 37de4e9..a0277a0 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "sqlseal", "name": "SQLSeal", - "version": "0.17.0", + "version": "0.18.0", "minAppVersion": "0.15.0", "description": "Use SQL in your notes to query your vault files and CSV content.", "author": "hypersphere", diff --git a/package.json b/package.json index 79f8834..8a7f076 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sqlseal", - "version": "0.17.0", + "version": "0.18.0", "description": "A plugin for Obsidian that allows you to run SQL queries on your notes.", "main": "main.js", "scripts": { diff --git a/src/codeblockHandler/CodeblockProcessor.ts b/src/codeblockHandler/CodeblockProcessor.ts index 3b09ed9..09afe36 100644 --- a/src/codeblockHandler/CodeblockProcessor.ts +++ b/src/codeblockHandler/CodeblockProcessor.ts @@ -2,7 +2,8 @@ import { OmnibusRegistrator } from "@hypersphere/omnibus"; import { App, MarkdownPostProcessorContext, MarkdownRenderChild, TFile } from "obsidian"; import { SqlSealDatabase } from "src/database/database"; import { Sync } from "src/datamodel/sync"; -import { parseLanguage, Table } from "src/grammar/newParser"; +import { ParserTableDefinition } from "src/datamodel/syncStrategy/types"; +import { parseLanguage } from "src/grammar/newParser"; import { RendererRegistry, RenderReturn } from "src/renderer/rendererRegistry"; import { transformQuery } from "src/sql/sqlTransformer"; import { registerObservers } from "src/utils/registerObservers"; @@ -33,7 +34,7 @@ export class CodeblockProcessor extends MarkdownRenderChild { if (results.tables) { await this.registerTables(results.tables) if (!results.queryPart) { - displayNotice(this.el, `Creating tables: ${results.tables.map(t => t.tableName).join(', ')}`) + displayNotice(this.el, `Creating tables: ${results.tables.map(t => t.tableAlias).join(', ')}`) return } } @@ -80,29 +81,7 @@ export class CodeblockProcessor extends MarkdownRenderChild { } } - async registerTables(tables: Table[]) { - await Promise.all(tables.map((table) => { - let path: TFile | null = null - let extras = {} - if (table.type !== 'table') { - path = this.app.metadataCache.getFirstLinkpathDest(table.fileName, this.ctx.sourcePath) - - } else { - path = this.app.vault.getFileByPath(this.ctx.sourcePath) - extras = table?.extras ?? {} - } - - if (!path) { - throw new Error(`File does not exist: ${table.fileName} (for ${table.tableName}).`) - } - - return this.sync.registerTable({ - aliasName: table.tableName, - type: table.type, - fileName: path.path, - sourceFile: this.ctx.sourcePath, - extras - }) - })) + async registerTables(tables: ParserTableDefinition[]) { + await Promise.all(tables.map((table) => this.sync.registerTable(table))) } } \ No newline at end of file diff --git a/src/database/database.ts b/src/database/database.ts index 7ef74ac..abba5c5 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -1,41 +1,8 @@ import { App } from "obsidian" -import { FieldTypes, toTypeStatements } from "../utils/typePredictions" import * as Comlink from 'comlink' import workerCode from 'virtual:worker-code' import { WorkerDatabase } from "./worker/database"; -export interface FieldDefinition { - name: string; - type: FieldTypes -} - -const formatData = (data: Record) => { - return Object.keys(data).reduce((ret, key) => { - if (typeof data[key] === 'boolean') { - return { - ...ret, - [key]: data[key] ? 1 : 0 - } - } - if (!data[key]) { - return { - ...ret, - [key]: null - } - } - if (typeof data[key] === 'object' || Array.isArray(data[key])) { - return { - ...ret, - [key]: JSON.stringify(data[key]) - } - } - return { - ...ret, - [key]: data[key] - } - }, {}) -} - export class SqlSealDatabase { db: Comlink.Remote private isConnected = false @@ -88,14 +55,6 @@ export class SqlSealDatabase { await this.db.recreateDatabase() } - async createTableWithData(name: string, data: Array>) { - const schema = await this.getSchema(data) - await this.createTable(name, schema) - await this.insertData(name, data) - - return schema - } - async updateData(name: string, data: Array>) { return this.db.updateData(name, data) } @@ -112,26 +71,16 @@ export class SqlSealDatabase { return this.db.dropTable(name) } - async createTableClean(name: string, fields: Array) { - const fieldsToRecord = fields.reduce((acc, f) => ({ - ...acc, - [f.name]: f.type - }), {} as Record) as Record - await this.createTable(name, fieldsToRecord) - } - - async createTable(name: string, fields: Record, noDrop?: boolean) { - await this.db.createTable(name, fields, noDrop) - } - async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) { await this.db.createTableNoTypes(name, columns, noDrop) } - async getSchema(data: Array>) { - const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); - const { types } = toTypeStatements(fields, data as Array>); // Call the toTypeStatements function with the correct arguments - return types; + async getColumns(tableName: string) { + return this.db.getColumns(tableName) + } + + async addColumns(tableName: string, newColumns: string[]) { + return this.db.addColumns(tableName, newColumns) } async select(statement: string, frontmatter: Record) { diff --git a/src/database/worker/database.ts b/src/database/worker/database.ts index e1a19b0..ca799b1 100644 --- a/src/database/worker/database.ts +++ b/src/database/worker/database.ts @@ -5,7 +5,6 @@ import { SQLiteFS } from 'absurd-sql'; import IndexedDBBackend from '../../../node_modules/absurd-sql/dist/indexeddb-backend.js'; import type { BindParams, Database, Statement } from "sql.js"; import { sanitise } from "../../utils/sanitiseColumn"; -import { FieldTypes } from "../../utils/typePredictions"; import { uniq } from "lodash"; @@ -114,7 +113,7 @@ export class WorkerDatabase { } async recreateDatabase() { - this.db.exec(` + this.db.run(` PRAGMA writable_schema = 1; DELETE FROM sqlite_master; PRAGMA writable_schema = 0; @@ -123,24 +122,20 @@ export class WorkerDatabase { `) } - async createTable(tableName: string, fields: Record, noDrop: boolean = false) { - const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type]) - const uniqueFields = [...new Map(transformedFiels.map(item => - [item[0], item])).values()] - const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`) - // FIXME: probably use schema generator, for now create with hardcoded fields - if (!noDrop) { - await this.dropTable(tableName) - } - const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} ( - ${sqlFields.join(', ')} - );` + run(query: string, params?: BindParams) { + this.db.run(query, params) + } - this.db.prepare(createSQL).run() - if (!noDrop) { - await this.clearTable(tableName) + getColumns(tableName: string) { + const [data] = this.db.exec('select name from pragma_table_info(@tableName)', { '@tableName': tableName }) + return data.values.map(d => d[0] as unknown as string) + } + + addColumns(tableName: string, newColumns: string[]) { + for (const columnName of newColumns) { + const stmt = `ALTER TABLE ${tableName} ADD COLUMN ${columnName}` + this.db.run(stmt) } - // Dropping data. } /* Types are optional in SQLite, we can take advantage of that */ @@ -150,11 +145,11 @@ export class WorkerDatabase { await this.dropTable(tableName) } const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fields.join(',')})` - this.db.prepare(createStmt).run() + this.db.run(createStmt) } async clearTable(tableName: string) { - this.db.prepare(`DELETE FROM ${tableName}`).run() + this.db.run(`DELETE FROM ${tableName}`) } @@ -163,6 +158,7 @@ export class WorkerDatabase { const columns = Object.keys(d).filter(c => c !== '__parsed_extra') const insertStatement = this.db.prepare(`INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`); insertStatement.run(recordToBindParams(formatData(d))) + insertStatement.free() }) } @@ -172,10 +168,11 @@ export class WorkerDatabase { async select(query: string, params: Record) { const stmt = this.db.prepare(query, recordToBindParams(params)) - return { - data: toObjectArray(stmt), - columns: stmt.getColumnNames() - } + const data = toObjectArray(stmt) + const columns = stmt.getColumnNames() + stmt.free() + + return { data, columns } } async updateData(tableName: string, data: Array>, matchKey: string = 'id') { @@ -183,6 +180,7 @@ export class WorkerDatabase { data.forEach((d: Record) => { const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE ${matchKey} = @${matchKey}`) stmt.run(recordToBindParams(d)) + stmt.free() }) } @@ -192,6 +190,7 @@ export class WorkerDatabase { stmt.run({ [`@${key}`]: d[key] } as BindParams) + stmt.free() }) } diff --git a/src/datamodel/repository/fileLog.ts b/src/datamodel/repository/fileLog.ts deleted file mode 100644 index e63ae96..0000000 --- a/src/datamodel/repository/fileLog.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { SourceType } from "src/grammar/newParser"; -import { Repository } from "./abstractRepository"; -import { v4 as uuidv4 } from "uuid"; - -export interface FileLog { - id: string, - table_name: string, - file_name: string, - created_at: string, - updated_at: string, - file_hash: string, - type: SourceType, - extras: Record -} - -export class FileLogRepository extends Repository { - - async init() { - await this.createTable() - } - - private async createTable() { - await this.db.createTable('file_log', { - 'id': 'TEXT', - 'table_name': 'TEXT', - 'file_name': 'TEXT', - 'created_at': 'TEXT', - 'updated_at': 'TEXT', - 'file_hash': 'TEXT', - 'type': 'TEXT', - extras: 'TEXT' - }, true) - } - - async insert(log: Omit) { - const now = Date.now() - await this.db.insertData('file_log', [{ - id: uuidv4(), - table_name: log.table_name, - file_name: log.file_name, - file_hash: log.file_hash, - type: log.type, - extras: JSON.stringify(log.extras ?? {}), - created_at: now, - updated_at: now - }]) - } - - async getByFilename(filename: string) { - const { data } = await this.db.select('SELECT * FROM file_log WHERE file_name = @filename', { - filename: filename - }) - - if (!data.length) { - return null - } - const d = data[0] - return { - ...d, - type: d.type ?? 'file', - extras: JSON.parse(d.extras ? d.extras.toString() : '{}') - } as FileLog - } - - async getAll() { - const { data } = await this.db.select('SELECT * FROM file_log', {}) - return data.map(d => ({ - ...d, - type: d.type ?? 'file', - extras: JSON.parse(d.extras ? d.extras.toString() : '{}') - })) as FileLog[] - } - - async updateHash(tableName: string, newHash: string) { - await this.db.db.updateData('file_log', [{ - table_name: tableName, - file_hash: newHash - }], 'table_name') - - } -} \ No newline at end of file diff --git a/src/datamodel/repository/tableMapLog.ts b/src/datamodel/repository/tableAliases.ts similarity index 60% rename from src/datamodel/repository/tableMapLog.ts rename to src/datamodel/repository/tableAliases.ts index 3419935..6f22266 100644 --- a/src/datamodel/repository/tableMapLog.ts +++ b/src/datamodel/repository/tableAliases.ts @@ -1,7 +1,7 @@ import { Repository } from "./abstractRepository"; import { v4 as uuid4 } from "uuid"; -export interface TableMapLog { +export interface TableAlias { id: string; table_name: string; alias_name: string; @@ -10,31 +10,28 @@ export interface TableMapLog { updated_at: string; } -export class TableMapLogRepository extends Repository { +export class TableAliasesRepository extends Repository { + + private readonly TABLE_NAME = 'TABLE_ALIASES'; async init() { await this.createTable() } async deleteMapping(id: string) { - this.db.deleteData('table_map_log', [{ id: id }], 'id') + this.db.deleteData(this.TABLE_NAME, [{ id: id }], 'id') } private async createTable() { // FIXME: add autoincrement index here to make it easier to manage. - await this.db.createTable('table_map_log', { - 'id': 'TEXT', - 'table_name': 'TEXT', - 'alias_name': 'TEXT', - 'source_file_name': 'TEXT', - 'created_at': 'TEXT', - 'updated_at': 'TEXT' - }, true) + await this.db.createTableNoTypes(this.TABLE_NAME,[ + 'id', 'table_name', 'alias_name', 'source_file_name', 'created_at', 'updated_at' + ], true) } - async insert(log: Omit) { + async insert(log: Omit) { const now = Date.now() - await this.db.insertData('table_map_log', [{ + await this.db.insertData(this.TABLE_NAME, [{ id: uuid4(), table_name: log.table_name, alias_name: log.alias_name, @@ -45,12 +42,12 @@ export class TableMapLogRepository extends Repository { } async getAll() { - const { data } = await this.db.select('SELECT * FROM table_map_log', {}) - return data as unknown as TableMapLog[] + const { data } = await this.db.select('SELECT * FROM TABLE_ALIASES', {}) + return data as unknown as TableAlias[] } async getByAlias(sourceFileName: string, aliasName: string) { - const { data } = await this.db.select(`SELECT * FROM table_map_log + const { data } = await this.db.select(`SELECT * FROM TABLE_ALIASES WHERE source_file_name=@source_file_name AND alias_name=@alias_name`, { 'source_file_name': sourceFileName, @@ -59,23 +56,23 @@ export class TableMapLogRepository extends Repository { if (!data || data.length < 0) { return null } - return data[0] as unknown as TableMapLog + return data[0] as unknown as TableAlias } async getByTableName(tableName: string) { - const { data } = await this.db.select(`SELECT * FROM table_map_log + const { data } = await this.db.select(`SELECT * FROM TABLE_ALIASES WHERE table_name = @table_name`, { table_name: tableName }) if (!data) { return [] } - return data as unknown[] as TableMapLog[] + return data as unknown[] as TableAlias[] } async getByContext(sourceFileName: string) { - const { data } = await this.db.select(`SELECT * FROM table_map_log + const { data } = await this.db.select(`SELECT * FROM TABLE_ALIASES WHERE source_file_name=@source_file_name `, { source_file_name: sourceFileName }) if (!data) { diff --git a/src/datamodel/repository/tableDefinitions.ts b/src/datamodel/repository/tableDefinitions.ts new file mode 100644 index 0000000..7ba23aa --- /dev/null +++ b/src/datamodel/repository/tableDefinitions.ts @@ -0,0 +1,106 @@ +import { Repository } from "./abstractRepository"; +import { v4 as uuidv4 } from "uuid"; + +export interface TableDefinition { + id: string, + table_name: string, + source_file: string, + created_at: string, + updated_at: string, + file_hash: string, + type: string, + refresh_id: string, // FIXME: we probably don't need this one, it is all solved on the table_name level. + arguments: string[] +} + +const parseDbEntry = (entry: TableDefinition): TableDefinition => ({ + ...entry, + arguments: JSON.parse(entry.arguments ? entry.arguments.toString() : '[]') +}) + +export type TableDefinitionExternal = Omit + +export class TableDefinitionsRepository extends Repository { + private readonly TABLE_NAME = 'TABLE_DEFINITIONS'; + + async init() { + await this.createTable() + } + + private async createTable() { + await this.db.createTableNoTypes( + this.TABLE_NAME, + ['id', 'table_name', 'source_file', 'created_at', 'updated_at', 'file_hash', 'type', 'refresh_id', 'arguments'], + true + ) + } + + async insert(definition: TableDefinitionExternal) { + const now = Date.now() + await this.db.insertData(this.TABLE_NAME, [{ + id: uuidv4(), + table_name: definition.table_name, + source_file: definition.source_file, + file_hash: definition.file_hash, + type: definition.type, + refresh_id: definition.refresh_id, + arguments: JSON.stringify(definition.arguments ?? []), + created_at: now, + updated_at: now + }]) + } + + async getBySourceFile(sourceFile: string) { + const { data } = await this.db.select( + `SELECT * FROM ${this.TABLE_NAME} WHERE source_file = @sourceFile`, + { sourceFile } + ) + + if (!data.length) { + return null + } + return (data as unknown[] as TableDefinition[]).map(parseDbEntry) + } + + async getByRefreshId(refreshId: string) { + const { data } = await this.db.select( + `SELECT * FROM ${this.TABLE_NAME} WHERE refresh_id= @refreshId`, + { refreshId } + ) + + if (!data.length) { + return null + } + const d = data[0] + return { + ...d, + arguments: JSON.parse(d.arguments ? d.arguments.toString() : '[]') + } as TableDefinition + } + + async getAll() { + const { data } = await this.db.select(`SELECT * FROM ${this.TABLE_NAME}`, {}) + return data.map(d => ({ + ...d, + type: d.type ?? 'file', + arguments: JSON.parse(d.arguments ? d.arguments.toString() : '[]') + })) as TableDefinition[] + } + + async update(id: string, fields: Partial) { + const updateData: Record = { + ...fields, + updated_at: Date.now().toString() + }; + + // Handle arguments field if it exists in the update + if ('arguments' in fields) { + updateData.arguments = JSON.stringify(fields.arguments); + } + + await this.db.db.updateData(this.TABLE_NAME, [{ + id, + ...updateData + }], 'id') + } +} \ No newline at end of file diff --git a/src/datamodel/sync.ts b/src/datamodel/sync.ts index 954a46c..cbedd6e 100644 --- a/src/datamodel/sync.ts +++ b/src/datamodel/sync.ts @@ -1,23 +1,22 @@ import { SqlSealDatabase } from "src/database/database"; -import { FileLog, FileLogRepository } from "./repository/fileLog"; -import { TableMapLogRepository } from "./repository/tableMapLog"; +import { TableAliasesRepository } from "./repository/tableAliases"; import { App, TAbstractFile, TFile, Vault } from "obsidian"; import { FilepathHasher } from "../utils/hasher"; import { Omnibus } from "@hypersphere/omnibus"; -import { TableRegistration } from "./types"; import { SyncStrategyFactory } from "./syncStrategy/SyncStrategyFactory"; import { ConfigurationRepository } from "./repository/configuration"; +import { TableDefinitionsRepository, TableDefinition } from "./repository/tableDefinitions"; +import { ParserTableDefinition } from "./syncStrategy/types"; +import { uniq } from "lodash"; -const SQLSEAL_DATABASE_VERSION = 1; +const SQLSEAL_DATABASE_VERSION = 2; export class Sync { - private fileLog: FileLogRepository; - private tableMapLog: TableMapLogRepository; + private tableDefinitionsRepo: TableDefinitionsRepository; + private tableMapLog: TableAliasesRepository; private bus = new Omnibus() - private tableMaps = new Map() - constructor( private readonly db: SqlSealDatabase, private readonly vault: Vault, @@ -26,14 +25,6 @@ export class Sync { } - async updateTableMaps() { - const data = await this.fileLog.getAll() - this.tableMaps.clear() - data.forEach(log => { - this.tableMaps.set(log.file_name, log) - }) - } - async init() { await this.db.connect() @@ -52,21 +43,20 @@ export class Sync { await config.init() - this.fileLog = new FileLogRepository(this.db) - await this.fileLog.init() + this.tableDefinitionsRepo = new TableDefinitionsRepository(this.db) + await this.tableDefinitionsRepo.init() - this.tableMapLog = new TableMapLogRepository(this.db) + this.tableMapLog = new TableAliasesRepository(this.db) await this.tableMapLog.init() await config.setConfig('version', SQLSEAL_DATABASE_VERSION) - const fileLogs = await this.fileLog.getAll() - for (const log of fileLogs) { - await this.syncFileByName(log.file_name) + const fileLogs = await this.tableDefinitionsRepo.getAll() + const uniquePaths = uniq(fileLogs.map(l => l.source_file)) + for (const path of uniquePaths) { + await this.syncFileByName(path) } - await this.updateTableMaps() - // START SYNCING this.startSync() } @@ -82,22 +72,28 @@ export class Sync { async syncFile(file: TFile) { this.bus.trigger('file::change::'+file.path) + const maps = await this.tableDefinitionsRepo.getBySourceFile(file.path) - const entry = this.tableMaps.get(file.path) - if (!entry) { + if (!maps) { return } - if (entry.file_hash !== file.stat.mtime.toString()) { - const syncObject = SyncStrategyFactory.getStrategyByFileLog(entry, this.app) - - const { data, columns } = await syncObject.returnData() - - const tableName = entry.table_name - await this.db.createTableClean(tableName, columns) - await this.db.insertData(tableName, data) - await this.fileLog.updateHash(tableName, file.stat.mtime.toString()) - this.bus.trigger('change::' + tableName) + for (const entry of maps) { + if (!entry) { + return + } + if (entry.file_hash !== file.stat.mtime.toString()) { + const syncObject = SyncStrategyFactory.getStrategy(entry, this.app) + + const { data, columns } = await syncObject.returnData() + + const tableName = entry.table_name + await this.db.createTableNoTypes(tableName, columns) + await this.db.insertData(tableName, data) + await this.tableDefinitionsRepo.update(entry.id, { file_hash: file.stat.mtime.toString() }) + this.bus.trigger('change::' + tableName) + + } } } @@ -109,9 +105,8 @@ export class Sync { }) } - async registerFile(log: Omit) { - await this.fileLog.insert(log) - await this.updateTableMaps() + async registerFile(log: Omit) { + await this.tableDefinitionsRepo.insert(log) } async getTablesMappingForContext(sourceFileName: string) { @@ -134,43 +129,42 @@ export class Sync { return `file_${hash}` } - async registerTable(reg: TableRegistration) { - const existingFileLog = await this.fileLog.getByFilename(reg.fileName) - let fileTableName: string; - const syncObject = SyncStrategyFactory.getStrategy(reg, this.app) - if (!existingFileLog) { - // We need to create new one. - const tableName = await syncObject.tableName() - await this.registerFile({ - file_name: reg.fileName, - table_name: tableName, - file_hash: '', - type: reg.type, - extras: reg.extras - }) - fileTableName = tableName - // INITIAL SYNC - await this.syncFileByName(reg.fileName) + async registerTable(reg: ParserTableDefinition) { + const syncObject = await SyncStrategyFactory.getStrategyFromParser(reg, this.app) + + const definition = syncObject.tableDefinition + + const existingDefinition = await this.tableDefinitionsRepo.getByRefreshId(definition.refresh_id) // FIXME: probably can be by table name too. + + let tableName: string; + + if (!existingDefinition) { + // This one is not registered yet, registering + await this.registerFile(definition) + await this.syncFileByName(definition.source_file) + tableName = definition.table_name } else { - fileTableName = existingFileLog.table_name + tableName = existingDefinition.table_name } - const existingTableLog = await this.tableMapLog.getByAlias(reg.sourceFile, reg.aliasName) + // TODO: THIS PART SHOULD BE REWRITTEN SOONish + const existingTableLog = await this.tableMapLog.getByAlias(reg.sourceFile, reg.tableAlias) if (!existingTableLog) { + // console.log(`Registering new mapping ${reg.sourceFile} :: ${reg.tableAlias} -> ${tableName}`) // Create new one await this.tableMapLog.insert({ - alias_name: reg.aliasName, + alias_name: reg.tableAlias, source_file_name: reg.sourceFile, - table_name: fileTableName, + table_name: tableName, }) } else { - // Already exists, doing nothing. // Check if it is the same mapping - if (existingTableLog.table_name != fileTableName) { - await this.tableMapLog.deleteMapping(existingTableLog.id) // FIXME: might need to remove event too + if (existingTableLog.table_name !== tableName) { + // console.log(`Alias ${reg.sourceFile} :: ${reg.tableAlias} changed table, now it should refer to ${tableName})`) + await this.tableMapLog.deleteMapping(existingTableLog.id) await this.tableMapLog.insert({ - alias_name: reg.aliasName, - table_name: fileTableName, + alias_name: reg.tableAlias, + table_name: tableName, source_file_name: reg.sourceFile }) } @@ -188,22 +182,4 @@ export class Sync { } return `change::${log.table_name}` } - - // async onFileChange(file: TFile) { - // const path = file.path - // const hash = file.stat.mtime - // // Checking if the file is registered - // const reg = await this.fileLog.getByFilename(path) - // if (reg) { - // // Need to syncronise it. - // // Need to save update in the database. - // // Need to trigger all related files. - // const allRelated = this.tableMapLog.getByTableName(reg.tableName) - // for (const related of allRelated) { - // this.bus.bus.trigger('') - // } - // } - - // // Checking if the file file is in the map. - // } } \ No newline at end of file diff --git a/src/datamodel/syncStrategy/CsvFileSyncStrategy.ts b/src/datamodel/syncStrategy/CsvFileSyncStrategy.ts index e13062d..7439e57 100644 --- a/src/datamodel/syncStrategy/CsvFileSyncStrategy.ts +++ b/src/datamodel/syncStrategy/CsvFileSyncStrategy.ts @@ -1,47 +1,71 @@ -import { App } from "obsidian"; -import { TableRegistration } from "../types"; import { ISyncStrategy } from "./abstractSyncStrategy"; import { sanitise } from "../../utils/sanitiseColumn"; import { parse } from "papaparse"; -import { FieldTypes, toTypeStatements } from "../../utils/typePredictions"; import { FilepathHasher } from "../../utils/hasher"; -import { TableDefinitionConfig } from "./types"; -import { SourceType } from "../../grammar/newParser"; +import { TableDefinitionExternal } from "../repository/tableDefinitions"; +import { ParserTableDefinition } from "./types"; +import { App } from "obsidian"; -export class CsvFileSyncStrategy implements ISyncStrategy { - constructor(private reg: TableRegistration, private app: App) { +const DEFAULT_FILE_HASH = '' + +export class CsvFileSyncStrategy extends ISyncStrategy { + + + static async fromParser(def: ParserTableDefinition, app: App): Promise { + + const tableSourceFile = def.arguments[0] + + const path = app.metadataCache.getFirstLinkpathDest(tableSourceFile, def.sourceFile) + if (!path) { + throw new Error(`File not found: ${tableSourceFile}`) + } + const sourcePath = path.path + const hash = await FilepathHasher.sha256(`${sourcePath}`) // FILENAME is in this case + const tableName = `file_${hash}` + + return new CsvFileSyncStrategy({ + arguments: def.arguments, + file_hash: DEFAULT_FILE_HASH, + refresh_id: tableName, + source_file: sourcePath, + table_name: tableName, + type: def.type + }, app) } - async tableName() { - const hash = await FilepathHasher.sha256(`${this.reg.sourceFile}`) // FILENAME is in this case - return `file_${hash}` - } - - static processTableDefinition(config: TableDefinitionConfig) { + async toTableDefinition(): Promise { + const tableSourceFile = this.def.arguments[0] + const hash = await FilepathHasher.sha256(`${tableSourceFile}`) // FILENAME is in this case + const def = this.def + const tableName = `file_${hash}` return { - tableName: config.alias, - type: config.type as SourceType, - fileName: config.arguments + arguments: def.arguments, + file_hash: DEFAULT_FILE_HASH, + refresh_id: tableName, + source_file: tableSourceFile, + table_name: tableName, + type: def.type } } async returnData() { - const file = this.app.vault.getFileByPath(this.reg.sourceFile)! + const file = this.app.vault.getFileByPath(this.def.source_file)! const data = await this.app.vault.cachedRead(file) + // TODO: PROBABLY SHOULD BE EXTRACTED SOMEWHERE FROM HERE later. const parsed = parse>(data, { header: true, - dynamicTyping: false, + dynamicTyping: true, skipEmptyLines: true, transformHeader: sanitise }) - const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data) - const columns = Object.entries(typeStatements.types).map(([key, value]) => ({ - name: key, - type: value as FieldTypes - })); + // const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data) + // const columns = Object.entries(typeStatements.types).map(([key, value]) => ({ + // name: key, + // type: value as FieldTypes + // })); - return { columns, data: parsed.data } + return { data: parsed.data, columns: parsed.meta.fields ?? [] } } } \ No newline at end of file diff --git a/src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts b/src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts index a81844a..a2375a3 100644 --- a/src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts +++ b/src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts @@ -1,44 +1,32 @@ import { FilepathHasher } from "../../utils/hasher"; -import { TableRegistration } from "../types"; import { ISyncStrategy } from "./abstractSyncStrategy"; import type { App } from "obsidian"; import { Component, MarkdownRenderer } from "obsidian"; -import { FieldTypes, toTypeStatements } from "../../utils/typePredictions"; -import { TableDefinitionConfig } from "./types"; -import { SourceType } from "../../grammar/newParser"; +import { ParserTableDefinition } from "./types"; + +const DEFAULT_FILE_HASH = '' -interface TableData { - columns: string[]; - data: Array>; -} +export class MarkdownTableSyncStrategy extends ISyncStrategy { -export class MarkdownTableSyncStrategy implements ISyncStrategy { - constructor(private reg: TableRegistration, private app: App) { - - } - - static processTableDefinition(config: TableDefinitionConfig) { - return { - tableName: config.alias, - type: config.type as SourceType, - fileName: config.sourceFile, //+ '?table=' + config.arguments - extras: { - tableNo: parseInt(config.arguments, 10) ?? 0 - } - } - } - - async tableName() { - const hash = await FilepathHasher.sha256(`${this.reg.sourceFile}__${this.reg.fileName}`) // FILENAME is in this case - - return `mdtable_${hash}` + static async fromParser(def: ParserTableDefinition, app: App): Promise { + const index = def.arguments[0] + const hash = await FilepathHasher.sha256(`${def.sourceFile}`) // FILENAME is in this case + const tableName = `mdtable_${hash}_${index}` + return new MarkdownTableSyncStrategy({ + arguments: def.arguments, + file_hash: DEFAULT_FILE_HASH, + refresh_id: tableName, + source_file: def.sourceFile, + table_name: tableName, + type: def.type + }, app) } async returnData() { - const tableIndex = this.reg.extras.tableNo - const file = this.app.vault.getFileByPath(this.reg.sourceFile)! + const tableIndex = parseInt(this.def.arguments[0], 10) + const file = this.app.vault.getFileByPath(this.def.source_file)! const content = await this.app.vault.read(file); // Create a temporary div to render the markdown @@ -54,13 +42,13 @@ export class MarkdownTableSyncStrategy implements ISyncStrategy { // Check if the requested table exists if (tableIndex >= tables.length) { - return null; + return { data: [], columns: [] }; } const targetTable = tables[tableIndex]; // Extract headers - const headers = Array.from(targetTable.querySelectorAll('th')).map(th => th.textContent?.trim() || ''); + const headers = Array.from(targetTable.querySelectorAll('th')).map(th => th.textContent?.trim() || '').filter(x => !!x); // Extract rows const rows = Array.from(targetTable.querySelectorAll('tr')).slice(1); // Skip header row @@ -75,15 +63,9 @@ export class MarkdownTableSyncStrategy implements ISyncStrategy { }, {} as Record); }); - const typeStatements = toTypeStatements(headers ?? [], data) - const columns = Object.entries(typeStatements.types).map(([key, value]) => ({ - name: key, - type: value as FieldTypes - })); - return { - columns, - data + data, + columns: headers ?? [] }; } } \ No newline at end of file diff --git a/src/datamodel/syncStrategy/SyncStrategyFactory.ts b/src/datamodel/syncStrategy/SyncStrategyFactory.ts index b21984b..1a20a1e 100644 --- a/src/datamodel/syncStrategy/SyncStrategyFactory.ts +++ b/src/datamodel/syncStrategy/SyncStrategyFactory.ts @@ -1,24 +1,25 @@ import { App } from "obsidian"; -import { FileLog } from "../repository/fileLog"; -import { TableRegistration } from "../types"; import { ISyncStrategy } from "./abstractSyncStrategy"; import { CsvFileSyncStrategy } from "./CsvFileSyncStrategy"; import { MarkdownTableSyncStrategy } from "./MarkdownTableSyncStrategy"; import { JsonFileSyncStrategy } from "./JSONFileSyncStrategy"; +import { ParserTableDefinition } from "./types"; +import { TableDefinitionExternal } from "../repository/tableDefinitions"; -const fileLogToTableRegistration = (log: FileLog): TableRegistration => { - return { - aliasName: log.table_name, - extras: log.extras, - fileName: log.file_name, - sourceFile: log.file_name, // ???? probably not needed here - type: log.type - } +function getFileExtension(pathname: string): string | null { + + + // FIXME: splitting for now + return pathname.split('.')[1].toLowerCase() + + // const url = new URL(pathname); + // const filePath = url.pathname; + // const fileName = path.basename(filePath); + // return path.extname(fileName); } const resolveFileStrategy = (filename: string) => { - const parts = filename.split('.') - const extension = parts[parts.length - 1].toLowerCase() + const extension = getFileExtension(filename) switch (extension) { case 'csv': @@ -31,27 +32,25 @@ const resolveFileStrategy = (filename: string) => { } } +const resolveClass = (type: string, filename: string) => { + switch (type) { + case 'file': + return resolveFileStrategy(filename) + case 'table': + return MarkdownTableSyncStrategy + default: + throw new Error(`Undefined strategy for type ${type}`) + } +} + export class SyncStrategyFactory { - static getStrategy(reg: TableRegistration, app: App): ISyncStrategy { - switch (reg.type) { - case 'file': - const Cls = resolveFileStrategy(reg.fileName) - return new Cls(reg, app) - case 'table': - return new MarkdownTableSyncStrategy(reg, app) - } + static async getStrategyFromParser(def: ParserTableDefinition, app: App): Promise { + const Cls = resolveClass(def.type, def.arguments[0]) + return Cls.fromParser(def, app) } - static getStaticStrategyReference(type: string) { - switch (type) { - case 'file': - return CsvFileSyncStrategy - case 'table': - return MarkdownTableSyncStrategy - } - } - - static getStrategyByFileLog(log: FileLog, app: App) { - return SyncStrategyFactory.getStrategy(fileLogToTableRegistration(log), app) + static getStrategy(def: TableDefinitionExternal, app: App): ISyncStrategy { + const Cls = resolveClass(def.type, def.source_file) + return new Cls(def, app) } } \ No newline at end of file diff --git a/src/datamodel/syncStrategy/abstractSyncStrategy.ts b/src/datamodel/syncStrategy/abstractSyncStrategy.ts index c500720..d56ad57 100644 --- a/src/datamodel/syncStrategy/abstractSyncStrategy.ts +++ b/src/datamodel/syncStrategy/abstractSyncStrategy.ts @@ -1,4 +1,20 @@ -export interface ISyncStrategy { - tableName(): Promise; - returnData(): any; +import { App } from "obsidian"; +import { TableDefinitionExternal } from "../repository/tableDefinitions"; +import { ParserTableDefinition } from "./types"; + +export abstract class ISyncStrategy { + constructor(protected def: TableDefinitionExternal, protected app: App) { + } + + get tableDefinition(): TableDefinitionExternal { + return this.def + } + abstract returnData(): Promise<{ + data: Record[], + columns: string[] + }>; + + static async fromParser(def: ParserTableDefinition, app: App): Promise { + throw new Error('Not Defined') + } } \ No newline at end of file diff --git a/src/datamodel/syncStrategy/types.ts b/src/datamodel/syncStrategy/types.ts index d8371c6..f901f6d 100644 --- a/src/datamodel/syncStrategy/types.ts +++ b/src/datamodel/syncStrategy/types.ts @@ -1,6 +1,6 @@ -export interface TableDefinitionConfig { +export interface ParserTableDefinition { type: string; - alias: string; - arguments: string; + tableAlias: string; + arguments: string[]; sourceFile: string; } \ No newline at end of file diff --git a/src/datamodel/types.ts b/src/datamodel/types.ts deleted file mode 100644 index d0ad60e..0000000 --- a/src/datamodel/types.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { SourceType } from "src/grammar/newParser"; - -export interface TableRegistration { - fileName: string; - aliasName: string; - sourceFile: string; - type: SourceType; - extras: any; -} \ No newline at end of file diff --git a/src/editorExtension/inlineCodeBlock.ts b/src/editorExtension/inlineCodeBlock.ts index 60b7901..734e35d 100644 --- a/src/editorExtension/inlineCodeBlock.ts +++ b/src/editorExtension/inlineCodeBlock.ts @@ -3,7 +3,6 @@ import { RangeSetBuilder } from "@codemirror/state"; import { syntaxTree } from "@codemirror/language"; import { App } from "obsidian"; import { SqlSealDatabase } from "../database/database"; -import { RendererRegistry } from "../renderer/rendererRegistry"; import { Sync } from "../datamodel/sync"; import { SqlSealInlineHandler } from "src/codeblockHandler/inline/InlineCodeHandler"; import { InlineProcessor } from "src/codeblockHandler/inline/InlineProcessor"; diff --git a/src/grammar/parser.test.ts b/src/grammar/newParser.test.ts similarity index 59% rename from src/grammar/parser.test.ts rename to src/grammar/newParser.test.ts index b6868fa..b82bf04 100644 --- a/src/grammar/parser.test.ts +++ b/src/grammar/newParser.test.ts @@ -12,11 +12,12 @@ describe('Parser', () => { it('should parse properly table and select', () => { expect(parseLanguage(`TABLE x = file(a.csv) -SELECT * FROM files`)).toEqual({ +SELECT * FROM files`, "source.md")).toEqual({ tables: [{ - tableName: 'x', - fileName: 'a.csv', - type: 'file' + tableAlias: 'x', + arguments: ['a.csv'], + type: 'file', + sourceFile: "source.md" }], queryPart: 'SELECT * FROM files', intermediateContent: '' @@ -26,16 +27,18 @@ SELECT * FROM files`)).toEqual({ it('should parse properly multiple tables in the same file', () => { expect(parseLanguage(`TABLE x = file(a.csv) TABLE y = file(very-long-name.csv) -SELECT * FROM a JOIN y ON a.id=y.id`)).toEqual({ +SELECT * FROM a JOIN y ON a.id=y.id`, 's.md')).toEqual({ tables: [{ - tableName: 'x', - fileName: 'a.csv', - type: 'file' + tableAlias: 'x', + arguments: ['a.csv'], + type: 'file', + sourceFile: 's.md' }, { - tableName: 'y', - fileName: 'very-long-name.csv', - type: 'file' + tableAlias: 'y', + arguments: ['very-long-name.csv'], + type: 'file', + sourceFile: 's.md' }], queryPart: 'SELECT * FROM a JOIN y ON a.id=y.id', intermediateContent: '' @@ -44,22 +47,24 @@ SELECT * FROM a JOIN y ON a.id=y.id`)).toEqual({ it('should parse properly multiple tables in the same file', () => { expect(parseLanguage(`TABLE x = file(a.csv) -TABLE y = file(very-long-name.csv) +TABLE y = file( very-long-name.csv ) PLOT { x: 5, y: 654 } -SELECT * FROM a JOIN y ON a.id=y.id`)).toEqual({ +SELECT * FROM a JOIN y ON a.id=y.id`, 'source.md')).toEqual({ tables: [{ - tableName: 'x', - fileName: 'a.csv', - type: 'file' + tableAlias: 'x', + arguments: ['a.csv'], + type: 'file', + sourceFile: 'source.md' }, { - tableName: 'y', - fileName: 'very-long-name.csv', - type: 'file' + tableAlias: 'y', + arguments: ['very-long-name.csv'], + type: 'file', + sourceFile: 'source.md' }], queryPart: 'SELECT * FROM a JOIN y ON a.id=y.id', intermediateContent: `PLOT { @@ -106,14 +111,15 @@ sElEcT * fRoM files`)).toEqual({ select * from files - `)).toEqual({ + `, 'src.md')).toEqual({ 'intermediateContent': 'html', 'queryPart': 'select * from files', tables: [ { - tableName: 'x', - fileName: 'a.csv', - type: 'file' + tableAlias: 'x', + arguments: ['a.csv'], + type: 'file', + sourceFile: 'src.md' } ] }) @@ -122,14 +128,32 @@ sElEcT * fRoM files`)).toEqual({ it('should properly parse query with just table and sql syntax', () => { const q = `table x = file(aaa.csv) select * from x` - expect(parseLanguage(q)).toEqual({ + expect(parseLanguage(q, '2025-01-01.md')).toEqual({ queryPart: 'select * from x', intermediateContent: '', tables: [ { - tableName: 'x', - fileName: 'aaa.csv', - type: 'file' + tableAlias: 'x', + arguments: ['aaa.csv'], + type: 'file', + sourceFile: '2025-01-01.md' + } + ] + }) + }) + + it('should properly parse query with table definition with multiple arguments', () => { + const q = `table x = file(aaa.csv, secondArg, third argument) +select * from x` + expect(parseLanguage(q, '2025-01-01.md')).toEqual({ + queryPart: 'select * from x', + intermediateContent: '', + tables: [ + { + tableAlias: 'x', + arguments: ['aaa.csv', 'secondArg', 'third argument'], + type: 'file', + sourceFile: '2025-01-01.md' } ] }) diff --git a/src/grammar/newParser.ts b/src/grammar/newParser.ts index 63350fc..74a4ceb 100644 --- a/src/grammar/newParser.ts +++ b/src/grammar/newParser.ts @@ -1,21 +1,7 @@ -import { SyncStrategyFactory } from "../datamodel/syncStrategy/SyncStrategyFactory"; -import { TableDefinitionConfig } from "../datamodel/syncStrategy/types"; - -export type SourceType = 'file' | 'table' - -export interface Table { - tableName: string; - type: SourceType, - fileName: string; - extras?: Record -} - -export interface TableWithParentPath extends Table { - parentPath: string; -} +import { ParserTableDefinition } from "../datamodel/syncStrategy/types"; interface ParsedLanguage { - tables: Table[]; + tables: ParserTableDefinition[]; queryPart: string; intermediateContent: string; } @@ -48,21 +34,15 @@ export function parseLanguage(input: string, sourceFile: string = ''): ParsedLan // Parse TABLE declaration // Format: TABLE tableName = file(filename.csv) - const tableMatch = line.match(/TABLE\s+(\w+)\s*=\s*(file|table)\(([^)]+)\)/i); + const tableMatch = line.match(/TABLE\s+(\w+)\s*=\s*(\w+)\(([^)]+)\)/i); if (tableMatch) { const config = { - alias: tableMatch[1], + tableAlias: tableMatch[1], type: tableMatch[2].toLowerCase(), - arguments: tableMatch[3], + arguments: tableMatch[3].split(',').map(t => t.trim()), sourceFile: sourceFile - } satisfies TableDefinitionConfig - const definition = SyncStrategyFactory - .getStaticStrategyReference(config.type) - ?.processTableDefinition(config) - if (!definition) { - throw new Error(`Cannot process table for: ${line}`) - } - result.tables.push(definition); + } satisfies ParserTableDefinition + result.tables.push(config); } currentPosition++; } diff --git a/src/utils/typePredictions.test.ts b/src/utils/typePredictions.test.ts deleted file mode 100644 index 17efb4c..0000000 --- a/src/utils/typePredictions.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { predictJson, predictType } from './typePredictions' - -describe('Utils', () => { - it('should properly predict TEXT type', () => { - expect(predictType('xxx', [{ 'xxx': 'dhjkafhkjafhkfas' }])).toEqual('TEXT') - }) - - it('should properly predict JSON type', () => { - expect(predictType('xxx', [{ 'xxx': { a: 'b' } }])).toEqual('JSON') - }) - - it('should properly predict json by parsing it', () => { - expect(predictJson([{ 'xxx': '["a", "b"]'}])).toEqual([{ - xxx: ['a', 'b'] - }]) - }) -}) \ No newline at end of file diff --git a/src/utils/typePredictions.ts b/src/utils/typePredictions.ts deleted file mode 100644 index 16b19a0..0000000 --- a/src/utils/typePredictions.ts +++ /dev/null @@ -1,99 +0,0 @@ - -import { parse } from 'json5' -import { camelCase } from "lodash"; - -export const predictType = (field: string, data: Array>) => { - - if (field === 'id') { - return 'TEXT' - } - - const d = data.map(d => d[field]) - - // CHECK JSONS. - const isObject = d.reduce((acc, d) => acc && (!d || typeof d === 'object'), true) - if (isObject) { - return 'JSON' - } - - const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field] as string), true) - if (canBeNumber) { - - // Check if Integer or Float - const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field] as string) === parseInt(d[field] as string), true) - if (canBeInteger) { - return 'INTEGER' - } - - return 'REAL' - } - return 'TEXT' -} - -export type FieldTypes = ReturnType - - -export const predictJson = (data: Array>) => { - return data.map(d => Object.keys(d).reduce((o, k) => { - if (typeof d[k] !== 'string') { - return { - ...o, - [k]: d[k] - } - } - const v = (d[k] as string).trim() - if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) { - try { - const d = parse(v) - return { - ...o, - [k]: d - } - } catch (e) { - } - } - return { - ...o, - [k]: d[k] - } - }, {})) -} - - -export function isNumeric(str: string) { - if (typeof str != "string") return false // we only process strings! - return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)... - !isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail -} - -export function dataToCamelCase(data: Array>) { - return data.map(d => Object.keys(d).reduce((acc, k) => ({ - ...acc, - [camelCase(k)]: d[k] - }), {})) -} - - -export const toTypeStatements = (header: Array, data: Array>) => { - let d: Array> = data - const types: Record> = {} - header.forEach(key => { - const type = predictType(key, data) - if (type === 'REAL' || type === 'INTEGER') { - // converting all data here to text - d = d.map(record => ({ - ...record, - [key]: type === 'REAL' - ? parseFloat(record[key] as string) - : parseInt(record[key] as string) - })) - } - - types[key] = type - }) - - return { - data: d, - types - } -} \ No newline at end of file diff --git a/src/utils/ui.ts b/src/utils/ui.ts index f16c403..ef9edbf 100644 --- a/src/utils/ui.ts +++ b/src/utils/ui.ts @@ -46,7 +46,7 @@ const generateLink = (config: SqlSealAnchorElement, app: App) => { if (fileHref.endsWith('.md')) { fileHref = fileHref.slice(0, -3) } - const vaultName = encodeURIComponent(app.appId); + const vaultName = encodeURIComponent((app as any).appId); const encodedPath = encodeURIComponent(fileHref); href = `obsidian://open?vault=${vaultName}&file=${encodedPath}`; } diff --git a/src/vaultSync/SealFileSync.ts b/src/vaultSync/SealFileSync.ts index 689ac65..fc37d61 100644 --- a/src/vaultSync/SealFileSync.ts +++ b/src/vaultSync/SealFileSync.ts @@ -1,5 +1,4 @@ import { App, Plugin, TFile } from "obsidian"; -import { FieldTypes } from "../utils/typePredictions"; import { sanitise } from "../utils/sanitiseColumn"; import { AFileSyncTable } from "./tables/abstractFileSyncTable"; @@ -13,7 +12,6 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => { } export class SealFileSync { - private currentSchema: Record = {} private tablePlugins: Array = [] constructor( public readonly app: App, @@ -23,14 +21,7 @@ export class SealFileSync { if (!(file instanceof TFile)) { return } - await sleep(100) - const frontmatter = await extractFrontmatterFromFile(file, plugin) - - if (this.hasNewColumns(frontmatter)) { - await sleep(1000) - await this.init() - return - } + await sleep(100) // FIXME: check if this is actually needed. await Promise.all(this.tablePlugins.map(p => p.onFileModify(file))) @@ -40,15 +31,6 @@ export class SealFileSync { if (!(file instanceof TFile)) { return } - const frontmatter = await extractFrontmatterFromFile(file, plugin) - - - if (this.hasNewColumns(frontmatter)) { - await sleep(1000) - await this.init() - - return - } await Promise.all(this.tablePlugins.map(p => p.onFileCreate(file))) })) @@ -101,11 +83,4 @@ export class SealFileSync { ) } - - hasNewColumns(newFrontmatter: Record) { - const currentFields = Object.keys(this.currentSchema) - const newFields = Object.keys(newFrontmatter).filter(f => !currentFields.includes(f)) - - return newFields.length > 0 - } } \ No newline at end of file diff --git a/src/vaultSync/tables/filesTable.ts b/src/vaultSync/tables/filesTable.ts index 30e2555..c229da0 100644 --- a/src/vaultSync/tables/filesTable.ts +++ b/src/vaultSync/tables/filesTable.ts @@ -2,10 +2,13 @@ import { App, Plugin, TAbstractFile, TFile } from "obsidian"; import { AFileSyncTable } from "./abstractFileSyncTable"; import { sanitise } from "../../utils/sanitiseColumn"; import { SqlSealDatabase } from "../../database/database"; -import { predictType } from "../../utils/typePredictions"; +import { difference } from "lodash"; -const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => { +export const FILES_TABLE_NAME = 'files' + + +const extractFrontmatterFromFile = (file: TFile, plugin: Plugin): Record => { const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {} return Object.fromEntries( @@ -14,7 +17,7 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => { ) } -function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record) { +function fileData(file: TAbstractFile, { tags: _tags, ...frontmatter }: Record) { return { ...frontmatter, path: file.path, @@ -24,41 +27,54 @@ function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record> + private columns: string[] = [] shouldPerformBulkInsert = true; constructor(db: SqlSealDatabase, app: App, private readonly plugin: Plugin) { super(db, app) } async onFileModify(file: TFile): Promise { // we need to update the row - const frontmatter = await extractFrontmatterFromFile(file, this.plugin) - await this.db.updateData('files', [fileData(file, frontmatter)]) + const frontmatter = extractFrontmatterFromFile(file, this.plugin) + const frontmatterWithFileData = fileData(file, frontmatter) + const columns = Object.keys(frontmatterWithFileData) + await this.updateColumnsIfNeeded(columns) + + await this.db.updateData(FILES_TABLE_NAME, []) await sleep(1000) // TO DELAY OTHER PLUGINS, UGLY BUT WORKS. } async onFileDelete(path: string): Promise { - await this.db.deleteData('files', [{ id: path }]) + await this.db.deleteData(FILES_TABLE_NAME, [{ id: path }]) + } + + async updateColumnsIfNeeded(newSetOfColumns: string[]) { + const newColumns = difference(newSetOfColumns, this.columns) + if (newColumns.length) { + await this.db.addColumns(FILES_TABLE_NAME, newColumns) + this.columns = await this.db.getColumns(FILES_TABLE_NAME) + } } async onFileCreate(file: TFile): Promise { - // we need to update the row - const frontmatter = await extractFrontmatterFromFile(file, this.plugin) - - // TODO: Check if there are new columns. If there are, we update the database. - - await this.db.insertData('files', [fileData(file, frontmatter)]) - - // FIXME: should we be waiting here? - await sleep(1000) + const frontmatter = extractFrontmatterFromFile(file, this.plugin) + const frontmatterWithFileData = fileData(file, frontmatter) + const columns = Object.keys(frontmatterWithFileData) + await this.updateColumnsIfNeeded(columns) + + await this.db.insertData(FILES_TABLE_NAME, [frontmatterWithFileData]) } async onFileCreateBulk(files: Array) { - // FIXME: implement this one and replace the other implementation. - const data = await Promise.all(files.map(async f => fileData(f, await extractFrontmatterFromFile(f, this.plugin)))) - const schema = await this.db.createTableWithData('files', data) - this.currentSchema = schema + this.columns = await this.db.getColumns(FILES_TABLE_NAME) + + // One by one + for (const file of files) { + await this.onFileCreate(file) + } } async onInit(): Promise { + this.db.createTableNoTypes(FILES_TABLE_NAME, ['id', 'name', 'path'], true) + this.columns = await this.db.getColumns(FILES_TABLE_NAME) } } \ No newline at end of file diff --git a/src/vaultSync/tables/tagsTable.ts b/src/vaultSync/tables/tagsTable.ts index c516e66..f4850f1 100644 --- a/src/vaultSync/tables/tagsTable.ts +++ b/src/vaultSync/tables/tagsTable.ts @@ -32,9 +32,6 @@ export class TagsFileSyncTable extends AFileSyncTable { async onInit(): Promise { - await this.db.createTable('tags', { - 'tag': 'TEXT', - 'fileId': 'TEXT' - }) + await this.db.createTableNoTypes('tags', ['tag', 'fileId']) } } \ No newline at end of file diff --git a/src/vaultSync/tables/tasksTable.ts b/src/vaultSync/tables/tasksTable.ts index 669b5c7..7baa274 100644 --- a/src/vaultSync/tables/tasksTable.ts +++ b/src/vaultSync/tables/tasksTable.ts @@ -45,10 +45,6 @@ export class TasksFileSyncTable extends AFileSyncTable { } async onInit(): Promise { - await this.db.createTable('tasks', { - 'task': 'TEXT', - 'completed': 'INTEGER', - 'filePath': 'TEXT' - }) + await this.db.createTableNoTypes('tasks', ['task', 'completed', 'filePath']) } } \ No newline at end of file diff --git a/versions.json b/versions.json index 9fb82f6..23960a5 100644 --- a/versions.json +++ b/versions.json @@ -25,5 +25,6 @@ "0.15.0": "0.15.0", "0.16.0": "0.15.0", "0.16.1": "0.15.0", - "0.17.0": "0.15.0" + "0.17.0": "0.15.0", + "0.18.0": "0.15.0" } \ No newline at end of file