From ed0c7bb74e5ad9d0e97108d885fdaa2734522b55 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 1 Sep 2024 14:01:54 +0100 Subject: [PATCH 1/4] many fixes: - fixing CSV loading - Fixing constantly reloading files when modifying sql inside the same block that has table declaration in - Auto-parsing JSON values with JSON5 - Better errors when error occurs in select statement --- main.ts | 13 ++-- package.json | 1 + pnpm-lock.yaml | 3 + src/SealFileSync.ts | 3 +- src/SealObserver.ts | 2 +- src/SqlSealCodeblockHandler.ts | 114 +++++++++++++++++++++++++++++++++ src/csvParse.ts | 7 -- src/database.ts | 102 +++++++++++++++++++---------- src/frontmatter.ts | 12 ++++ src/sqlSeal.ts | 99 ++-------------------------- src/utils.ts | 38 +++++++++++ tsconfig.json | 3 +- 12 files changed, 253 insertions(+), 144 deletions(-) create mode 100644 src/SqlSealCodeblockHandler.ts delete mode 100644 src/csvParse.ts create mode 100644 src/frontmatter.ts diff --git a/main.ts b/main.ts index 88f737b..0410adf 100644 --- a/main.ts +++ b/main.ts @@ -12,17 +12,15 @@ export default class SqlSealPlugin extends Plugin { async onload() { await this.loadSettings(); - const sqlSeal = new SqlSeal(this.app, false) + const sqlSeal = new SqlSeal(this.app, true) this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) - await sqlSeal.connect() - this.sqlSeal = sqlSeal - - - this.fileSync = new SealFileSync(this.app, sqlSeal, this) // start syncing when files are loaded this.app.workspace.onLayoutReady(() => { + sqlSeal.db.connect().then(() => { + this.fileSync = new SealFileSync(this.app, sqlSeal, this) this.fileSync.init() + }) }) // this.addSettingTab(new SqlSealSettingsTab(this.app, this)); @@ -30,9 +28,6 @@ export default class SqlSealPlugin extends Plugin { } onunload() { - if (this.fileSync) { - this.fileSync.destroy(); - } this.sqlSeal.db.disconect(); } diff --git a/package.json b/package.json index f92bff8..510d318 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ }, "dependencies": { "better-sqlite3": "^11.2.1", + "json5": "^2.2.3", "lodash": "^4.17.21", "node-sql-parser": "^5.0.0", "papaparse": "^5.4.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 978049a..62085a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: better-sqlite3: specifier: ^11.2.1 version: 11.2.1 + json5: + specifier: ^2.2.3 + version: 2.2.3 lodash: specifier: ^4.17.21 version: 4.17.21 diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 67e4ee5..e8e994e 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -1,5 +1,6 @@ import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian"; import { SqlSeal } from "./sqlSeal"; +import { FieldTypes } from "./utils"; function fileData(file: TAbstractFile, frontmatter: Record) { return { @@ -15,7 +16,7 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => { } export class SealFileSync { - private currentSchema: Record = {} + private currentSchema: Record = {} constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) { plugin.registerEvent(this.app.vault.on('modify', async (file) => { if (!(file instanceof TFile)) { diff --git a/src/SealObserver.ts b/src/SealObserver.ts index d7bfe25..b3cf2b6 100644 --- a/src/SealObserver.ts +++ b/src/SealObserver.ts @@ -27,7 +27,7 @@ export class SealObserver { if (observers !== undefined) { observers.add(observer); if (this.verbose) { - console.log(`Observer registered for table "${tableName}".`); + console.log(`Observer registered for "${tableName}".`); } } if (tag) { diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts new file mode 100644 index 0000000..2a2dc5b --- /dev/null +++ b/src/SqlSealCodeblockHandler.ts @@ -0,0 +1,114 @@ +import { App, MarkdownPostProcessorContext } from "obsidian" +import { displayData, displayError, displayInfo, displayLoader } from "./ui" +import { resolveFrontmatter } from "./frontmatter" +import { hashString } from "./hash" +import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables" +import { SealObserver } from "./SealObserver" +import { SqlSealDatabase } from "./database" + +export class SqlSealCodeblockHandler { + get globalTables() { + return ['files', 'tags'] + } + constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver) { } + + tablesConfig: Record = {} + + getHandler() { + return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => { + + displayLoader(el) + await this.db.connect() + this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers. + + const frontmatter = await resolveFrontmatter(ctx, this.app) + const prefix = hashString(ctx.sourcePath) + const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g; + let match + while ((match = regex.exec(source)) !== null) { + const name = match[1]; + const url = match[2]; + // UPDATING TABLES IF THEY CHANGED SINCE LAST REGISTER + const prefixedName = prefixedIfNotGlobal(name, this.globalTables, prefix) + if (this.tablesConfig[prefixedName] === url) { + // We do not need to rework it. Also, it should be watched, right? + // continue + } + + if (!!this.tablesConfig[prefixedName]) { + // We need to remove old database as it's pointing to the wrong collection (probably). + } + + + // FIXME: do not register observer when it's already been registed for this ctx. + this.observer.registerObserver(`file:${url}`, async () => { + console.log('UPDATE TABLE', name, url, prefix) + // Update table + await this.db.loadDataForDatabaseFromUrl(name, url, true) + + // Fire observers for the table + console.log('prefixed name', prefixedName) + this.observer.fireObservers(`table:${prefixedName}`) + }, ctx.docId) + if (this.tablesConfig[prefixedName] !== url) { + this.observer.fireObservers(`file:${url}`) + this.tablesConfig[prefixedName] = url + } else { + // We still need to update the table watchers? + requestAnimationFrame(() => { + this.observer.fireObservers(`table:${prefixedName}`) + }) + } + } + + const selectRegexp = /SELECT\s+(.*)/g; // OR WITH? + const selectMatch = selectRegexp.exec(source) + if (selectMatch) { + try { + // FIXME: WAIT FOR TABLES TO BE READDY!!!! + const selectStatement = selectMatch[0] + const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix) + + const renderSelect = async () => { + try { + const stmt = await this.db.db.prepare(statement) + const columns = await stmt.columns().map(column => column.name); + const data = await stmt.all(frontmatter ?? {}) + displayData(el, columns, data) + } catch (e) { + displayError(el, e) + } + } + + // Register observer for each table + tables.forEach(table => { + const observer = async () => { + if (!el.isConnected) { + // Unregistering using the context id + this.observer.unregisterObserversByTag(ctx.docId) + } + + + displayLoader(el) + + await renderSelect() + } + this.observer.registerObserver(`table:${table}`, observer, ctx.docId) + }) + + this.globalTables.forEach(t => this.observer.fireObservers(`table:${t}`)) + + // await renderSelect() <- this should get triggered automatically. + + } catch (e) { + if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) { + displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.') + } else { + console.error('e', e) + displayError(el, e) + } + } + } + } + } +} \ No newline at end of file diff --git a/src/csvParse.ts b/src/csvParse.ts deleted file mode 100644 index 6dba767..0000000 --- a/src/csvParse.ts +++ /dev/null @@ -1,7 +0,0 @@ -interface Results { - columns: Record, - data: Record[] -} -const parseCsv = (file) => { - -} \ No newline at end of file diff --git a/src/database.ts b/src/database.ts index 8e1fe1f..2e00bb5 100644 --- a/src/database.ts +++ b/src/database.ts @@ -4,20 +4,20 @@ import path from 'path' import Papa from 'papaparse' import { prefixedIfNotGlobal } from "./sqlReparseTables" import { camelCase } from 'lodash' -import { fetchBlobData, predictType } from "./utils" +import { fetchBlobData, FieldTypes, predictJson, predictType } from "./utils" import os from 'os' import fs from 'fs' export function isNumeric(str: string) { if (typeof str != "string") return false // we only process strings! - return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)... + return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)... !isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail } function dataToCamelCase(data: Array>) { return data.map(d => Object.keys(d).reduce((acc, k) => ({ ...acc, - [camelCase(k)]: Array.isArray(d[k]) ? d[k].join(', ') : d[k] + [camelCase(k)]: d[k] }), {})) } @@ -48,10 +48,10 @@ const toTypeStatements = (header: Array, data: Array = {} - db: typeof Database + db: Database.Database private isConnected = false - private connectingPromise; - private connectingPromiseResolve; + private connectingPromise: Promise; + private connectingPromiseResolve: (value: void | PromiseLike) => void constructor(private readonly app: App, private readonly verbose = false) { } @@ -168,24 +168,29 @@ export class SqlSealDatabase { return deleteMany(data) } - insertData(name: string, inData: Array>) { + async insertData(name: string, inData: Array>) { + console.log('IN DATA', inData) const data = dataToCamelCase(inData) const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); // FIXME: reworking all fields to be camel case if (!fields || !fields.length) { return } + console.log('DATA', data) const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); const insertMany = this.db.transaction((pData: Array>) => { pData.forEach(data => { try { // update data so all missing fields are set to null fields.forEach(field => { + if (field === 'events') { + console.log('GOT FIELD events', data[field], typeof data[field]) + } if (typeof data[field] === 'boolean') { data[field] = data[field] ? 1 : 0 } else if (!data[field]) { data[field] = null - } else if (typeof data[field] === 'object') { + } else if (typeof data[field] === 'object' || Array.isArray(data[field])) { data[field] = JSON.stringify(data[field]) } }) @@ -196,25 +201,26 @@ export class SqlSealDatabase { }) }) - return insertMany(data) + insertMany(data) } - async createTable(name: string, fields: Record) { + async createTable(name: string, fields: Record) { + console.log(`CREATE TABLE ${name}`) const transformedFiels = Object.entries(fields).map(([key, type]) => [camelCase(key), type]) const uniqueFields = [...new Map(transformedFiels.map(item => [item[0], item])).values()] const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`) // FIXME: probably use schema generator, for now create with hardcoded fields - await this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run() + this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run() const createSQL = `CREATE TABLE IF NOT EXISTS ${name} ( ${sqlFields.join(', ')} );` - await this.db.prepare(createSQL).run() + this.db.prepare(createSQL).run() this.savedDatabases[name] = true // Dropping data. - await this.db.prepare(`DELETE FROM ${name}`).run() + this.db.prepare(`DELETE FROM ${name}`).run() } async getSchema(data: Array>) { const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); @@ -222,36 +228,64 @@ export class SqlSealDatabase { return types; } - async loadDataForDatabaseFromUrl(name: string, url: string) { + async loadDataForDatabaseFromUrl(name: string, url: string, reloadData: boolean = false) { + console.log('SHOULD WE RELOAD ALL THE DATA?', reloadData) const file = this.app.vault.getFileByPath(url) + + if (!file) { + return + } const data = await this.app.vault.cachedRead(file) - const parsed = Papa.parse(data, { + const parsed = Papa.parse>(data, { header: true, dynamicTyping: false, skipEmptyLines: true }) - const fields = parsed.meta.fields - const { data: parsedData, types } = toTypeStatements(fields, parsed.data) - // Purge the database - await this.db.prepare(`DELETE FROM ${name}`).run() + const processedData = dataToCamelCase(parsed.data) + const processedWithJsonParsed = predictJson(processedData) + const fields = parsed.meta.fields?.map((f: string) => camelCase(f))! - const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); - const insertMany = this.db.transaction((pData: Array>) => { - pData.forEach(data => { - try { - insert.run(data) - } catch (e) { - console.error(e) - } - }) - }) + const { data: parsedData, types } = toTypeStatements(fields, processedWithJsonParsed) - await insertMany(parsedData) + try { + // Purge the database + await this.db.prepare(`DELETE FROM ${name}`).run() + } catch (e) { + // IF ERROR IS THAT TABLE DOES NOT EXIST, LETS CREATE ONE. + // FIXME: check if error is actually that the table does not exist + console.warn('Error', e) + await this.createTable(name, types) + } + await this.insertData(name, parsedData) + // const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); + // console.log(`INSERT STMT`, insert) + // const insertMany = this.db.transaction((pData: Array>) => { + // pData.forEach(data => { + // try { + // insert.run(data) + // } catch (e) { + // console.error(e) + // } + // }) + // }) + + // await insertMany(parsedData) + return name } + /** + * @deprecated + * @param unprefixedName + * @param url + * @param prefix + * @param reloadData + * @returns + */ async defineDatabaseFromUrl(unprefixedName: string, url: string, prefix: string, reloadData: boolean = false) { + // FIXME: why do we repeat this code? + console.log('define:: defineDatabaseFromUrl', unprefixedName, url) const name = prefixedIfNotGlobal(unprefixedName, [], prefix) // FIXME: should we pass global tables here too? if (this.savedDatabases[name]) { if (reloadData) { @@ -260,21 +294,23 @@ export class SqlSealDatabase { return name } const file = this.app.vault.getFileByPath(url) + + console.log('FILE', file) if (!file) { return name } const data = await this.app.vault.cachedRead(file) - const parsed = Papa.parse(data, { + const parsed = Papa.parse>(data, { header: true, dynamicTyping: false, skipEmptyLines: true }) - const fields = parsed.meta.fields + const fields = parsed.meta.fields! const { data: parsedData, types } = toTypeStatements(fields, parsed.data) await this.createTable(name, types) - // this.savedDatabases[name] = url + this.savedDatabases[name] = url await this.loadDataForDatabaseFromUrl(name, url) diff --git a/src/frontmatter.ts b/src/frontmatter.ts new file mode 100644 index 0000000..f3b7122 --- /dev/null +++ b/src/frontmatter.ts @@ -0,0 +1,12 @@ +import { App, MarkdownPostProcessorContext } from "obsidian" + +export const resolveFrontmatter = (ctx: Pick, app: App) => { + if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) { + return ctx.frontmatter as Record + } + const file = app.vault.getFileByPath(ctx.sourcePath) + if (!file) { + return null + } + return app.metadataCache.getFileCache(file)?.frontmatter +} \ No newline at end of file diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index d2f63d8..c82d091 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -1,113 +1,28 @@ import { SqlSealDatabase } from "./database"; -import { displayData, displayError, displayInfo, displayLoader } from "./ui"; -import { App, MarkdownPostProcessorContext } from "obsidian"; -import { updateTables } from "./sqlReparseTables"; -import { hashString } from "./hash"; +import { App } from "obsidian"; import { SealObserver } from "./SealObserver"; +import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler"; export class SqlSeal { public db: SqlSealDatabase public observer: SealObserver + public codeBlockHandler: SqlSealCodeblockHandler constructor(private readonly app: App, verbose = false) { this.db = new SqlSealDatabase(app, verbose) this.observer = new SealObserver(verbose) this.observeAllFileChanges() + this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer) + // FIXME: handle here changes to files and tags? } - public get globalTables() { - return ['files', 'tags'] + getHandler() { + return this.codeBlockHandler.getHandler() } - async connect() { - await this.db.connect() - } - - async disconnect() { - await this.db.disconect() - } - - async resolveFrontmatter(ctx: MarkdownPostProcessorContext) { - if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) { - return ctx.frontmatter as Record - } - const file = this.app.vault.getFileByPath(ctx.sourcePath) - if (!file) { - return null - } - return this.app.metadataCache.getFileCache(file)?.frontmatter - } - - private observeAllFileChanges() { // Use fs to observe file changes this.app.vault.on('modify', async (file) => { this.observer.fireObservers('file:' + file.path) }) } - - getHandler() { - return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => { - - displayLoader(el) - await this.connect() - - const frontmatter = await this.resolveFrontmatter(ctx) - const prefix = hashString(ctx.sourcePath) - const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g; - let match - while ((match = regex.exec(source)) !== null) { - const name = match[1]; - const url = match[2]; - const prefixedName = await this.db.defineDatabaseFromUrl(name, url, prefix) - this.observer.registerObserver(`file:${url}`, async () => { - // Update table - await this.db.defineDatabaseFromUrl(name, url, prefix, true) - - // Fire observers for the table - this.observer.fireObservers(`table:${prefixedName}`) - }, ctx.docId) - } - - const selectRegexp = /SELECT\s+(.*)/g; - const selectMatch = selectRegexp.exec(source) - if (selectMatch) { - try { - const selectStatement = selectMatch[0] - const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix) - - const renderSelect = async () => { - const stmt = await this.db.db.prepare(statement) - const columns = await stmt.columns().map(column => column.name); - const data = await stmt.all(frontmatter ?? {}) - displayData(el, columns, data) - } - - // Register observer for each table - tables.forEach(table => { - const observer = async () => { - if (!el.isConnected) { - // Unregistering using the context id - this.observer.unregisterObserversByTag(ctx.docId) - } - - - displayLoader(el) - - await renderSelect() - } - this.observer.registerObserver(`table:${table}`, observer, ctx.docId) - }) - - await renderSelect() - - } catch (e) { - if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) { - displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.') - } else { - displayError(el, e) - } - } - } - } - } } diff --git a/src/utils.ts b/src/utils.ts index d78b2b7..310cbdb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ import { get } from "https" +import JSON5 from 'json5' import * as fs from 'fs' import { isNumeric } from "./database"; @@ -77,3 +78,40 @@ export const predictType = (field: string, data: Array + + +export const predictJson = (data: Array>) => { + return data.map(d => Object.keys(d).reduce((o, k) => { + if (typeof d[k] !== 'string') { + return { + ...o, + [k]: d[k] + } + } + const v = d[k].trim() + if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) { + // We can try parsing + if (k === 'events') { + console.log(`JSON DATA: ${d[k]}`) + } + try { + const d = JSON5.parse(v) + if (k === 'events') { + console.log('PARSED', d) + } + return { + ...o, + [k]: d + } + } catch (e) { + console.log('NOT PARSED', e) + } + } + return { + ...o, + [k]: d[k] + } + }, {})) +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index db4fd63..0923291 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,7 +18,8 @@ "ES5", "ES6", "ES7" - ] + ], + "allowSyntheticDefaultImports": true }, "include": [ "**/*.ts" From 916ba5a4060f5c3913ecbc3357ef00d8d465d9e7 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 1 Sep 2024 17:04:48 +0100 Subject: [PATCH 2/4] chore: removing console.logs --- main.ts | 2 +- src/SqlSealCodeblockHandler.ts | 3 --- src/database.ts | 27 ++------------------------- src/utils.ts | 8 -------- 4 files changed, 3 insertions(+), 37 deletions(-) diff --git a/main.ts b/main.ts index 0410adf..f7aa43d 100644 --- a/main.ts +++ b/main.ts @@ -12,7 +12,7 @@ export default class SqlSealPlugin extends Plugin { async onload() { await this.loadSettings(); - const sqlSeal = new SqlSeal(this.app, true) + const sqlSeal = new SqlSeal(this.app, false) this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) this.sqlSeal = sqlSeal // start syncing when files are loaded diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts index 2a2dc5b..c712769 100644 --- a/src/SqlSealCodeblockHandler.ts +++ b/src/SqlSealCodeblockHandler.ts @@ -42,12 +42,10 @@ export class SqlSealCodeblockHandler { // FIXME: do not register observer when it's already been registed for this ctx. this.observer.registerObserver(`file:${url}`, async () => { - console.log('UPDATE TABLE', name, url, prefix) // Update table await this.db.loadDataForDatabaseFromUrl(name, url, true) // Fire observers for the table - console.log('prefixed name', prefixedName) this.observer.fireObservers(`table:${prefixedName}`) }, ctx.docId) if (this.tablesConfig[prefixedName] !== url) { @@ -104,7 +102,6 @@ export class SqlSealCodeblockHandler { if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) { displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.') } else { - console.error('e', e) displayError(el, e) } } diff --git a/src/database.ts b/src/database.ts index 2e00bb5..1bbb073 100644 --- a/src/database.ts +++ b/src/database.ts @@ -132,7 +132,8 @@ export class SqlSealDatabase { return } - const alter = this.db.prepare(`ALTER TABLE ${name} ADD COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`) + const alter = this.db.prepare(`ALTER TABLE ${name} ADD + COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`) alter.run() } @@ -169,23 +170,18 @@ export class SqlSealDatabase { } async insertData(name: string, inData: Array>) { - console.log('IN DATA', inData) const data = dataToCamelCase(inData) const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); // FIXME: reworking all fields to be camel case if (!fields || !fields.length) { return } - console.log('DATA', data) const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); const insertMany = this.db.transaction((pData: Array>) => { pData.forEach(data => { try { // update data so all missing fields are set to null fields.forEach(field => { - if (field === 'events') { - console.log('GOT FIELD events', data[field], typeof data[field]) - } if (typeof data[field] === 'boolean') { data[field] = data[field] ? 1 : 0 } else if (!data[field]) { @@ -205,7 +201,6 @@ export class SqlSealDatabase { } async createTable(name: string, fields: Record) { - console.log(`CREATE TABLE ${name}`) const transformedFiels = Object.entries(fields).map(([key, type]) => [camelCase(key), type]) const uniqueFields = [...new Map(transformedFiels.map(item => [item[0], item])).values()] @@ -229,7 +224,6 @@ export class SqlSealDatabase { } async loadDataForDatabaseFromUrl(name: string, url: string, reloadData: boolean = false) { - console.log('SHOULD WE RELOAD ALL THE DATA?', reloadData) const file = this.app.vault.getFileByPath(url) if (!file) { @@ -253,25 +247,10 @@ export class SqlSealDatabase { // Purge the database await this.db.prepare(`DELETE FROM ${name}`).run() } catch (e) { - // IF ERROR IS THAT TABLE DOES NOT EXIST, LETS CREATE ONE. // FIXME: check if error is actually that the table does not exist - console.warn('Error', e) await this.createTable(name, types) } await this.insertData(name, parsedData) - // const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); - // console.log(`INSERT STMT`, insert) - // const insertMany = this.db.transaction((pData: Array>) => { - // pData.forEach(data => { - // try { - // insert.run(data) - // } catch (e) { - // console.error(e) - // } - // }) - // }) - - // await insertMany(parsedData) return name } @@ -285,7 +264,6 @@ export class SqlSealDatabase { */ async defineDatabaseFromUrl(unprefixedName: string, url: string, prefix: string, reloadData: boolean = false) { // FIXME: why do we repeat this code? - console.log('define:: defineDatabaseFromUrl', unprefixedName, url) const name = prefixedIfNotGlobal(unprefixedName, [], prefix) // FIXME: should we pass global tables here too? if (this.savedDatabases[name]) { if (reloadData) { @@ -295,7 +273,6 @@ export class SqlSealDatabase { } const file = this.app.vault.getFileByPath(url) - console.log('FILE', file) if (!file) { return name } diff --git a/src/utils.ts b/src/utils.ts index 310cbdb..57e3797 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -92,21 +92,13 @@ export const predictJson = (data: Array>) => { } const v = d[k].trim() if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) { - // We can try parsing - if (k === 'events') { - console.log(`JSON DATA: ${d[k]}`) - } try { const d = JSON5.parse(v) - if (k === 'events') { - console.log('PARSED', d) - } return { ...o, [k]: d } } catch (e) { - console.log('NOT PARSED', e) } } return { From 28522d32b86ae92c3fcc95172b9e0baa2769f26a Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 1 Sep 2024 19:59:03 +0100 Subject: [PATCH 3/4] chore: fixing prefixing csv tables. --- main.ts | 2 +- src/SealObserver.ts | 22 ++++++++-------------- src/SqlSealCodeblockHandler.ts | 6 ++++-- src/logger.ts | 22 ++++++++++++++++++++++ src/sqlSeal.ts | 4 +++- 5 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 src/logger.ts diff --git a/main.ts b/main.ts index f7aa43d..22ed3f7 100644 --- a/main.ts +++ b/main.ts @@ -12,7 +12,7 @@ export default class SqlSealPlugin extends Plugin { async onload() { await this.loadSettings(); - const sqlSeal = new SqlSeal(this.app, false) + const sqlSeal = new SqlSeal(this.app, false) // FIXME: set verbose based on the env. this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) this.sqlSeal = sqlSeal // start syncing when files are loaded diff --git a/src/SealObserver.ts b/src/SealObserver.ts index b3cf2b6..8f62123 100644 --- a/src/SealObserver.ts +++ b/src/SealObserver.ts @@ -1,15 +1,17 @@ +import { Logger } from "./logger"; + type Callback = () => void; export class SealObserver { private tables: Map>; - private verbose: boolean; private tags: Map>; + private logger: Logger constructor(verbose = false) { this.tables = new Map(); this.tags = new Map(); - this.verbose = verbose; + this.logger = new Logger(verbose) } registerObserver(tableNames: string | string[], observer: Callback, tag?: string) { @@ -19,16 +21,12 @@ export class SealObserver { tableNames.forEach(tableName => { if (!this.tables.has(tableName)) { this.tables.set(tableName, new Set()); - if (this.verbose) { - console.log(`Table "${tableName}" registered.`); - } + this.logger.log(`Table "${tableName}" registered.`); } const observers = this.tables.get(tableName); if (observers !== undefined) { observers.add(observer); - if (this.verbose) { - console.log(`Observer registered for "${tableName}".`); - } + this.logger.log(`Observer registered for "${tableName}".`); } if (tag) { if (!this.tags.has(tag)) { @@ -58,9 +56,7 @@ export class SealObserver { this.tables.forEach(observers => { observers.delete(observer); }); - if (this.verbose) { - console.log(`Observer unregistered.`); - } + this.logger.log(`Observer unregistered.`); } fireObservers(tableName: string) { @@ -68,9 +64,7 @@ export class SealObserver { const observers = this.tables.get(tableName); if (observers) { observers.forEach(observer => observer()); - if (this.verbose) { - console.log(`Observers fired for table "${tableName}".`); - } + this.logger.log(`Observers fired for table "${tableName}".`); } } } diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts index c712769..d600046 100644 --- a/src/SqlSealCodeblockHandler.ts +++ b/src/SqlSealCodeblockHandler.ts @@ -5,12 +5,13 @@ import { hashString } from "./hash" import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables" import { SealObserver } from "./SealObserver" import { SqlSealDatabase } from "./database" +import { Logger } from "./logger" export class SqlSealCodeblockHandler { get globalTables() { return ['files', 'tags'] } - constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver) { } + constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) { } tablesConfig: Record = {} @@ -28,6 +29,7 @@ export class SqlSealCodeblockHandler { while ((match = regex.exec(source)) !== null) { const name = match[1]; const url = match[2]; + this.logger.log('CodedblockHandler. table', name, url) // UPDATING TABLES IF THEY CHANGED SINCE LAST REGISTER const prefixedName = prefixedIfNotGlobal(name, this.globalTables, prefix) if (this.tablesConfig[prefixedName] === url) { @@ -43,7 +45,7 @@ export class SqlSealCodeblockHandler { // FIXME: do not register observer when it's already been registed for this ctx. this.observer.registerObserver(`file:${url}`, async () => { // Update table - await this.db.loadDataForDatabaseFromUrl(name, url, true) + await this.db.loadDataForDatabaseFromUrl(prefixedName, url, true) // Fire observers for the table this.observer.fireObservers(`table:${prefixedName}`) diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 0000000..8e376d3 --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,22 @@ +export class Logger { + constructor(verbose: boolean) { + if (verbose) { + this.console = console + } else { + this.console = { + log: () => { }, + error: () => { } + } + } + } + + private console: Pick + + log(...args: any[]) { + this.console.log(...args) + } + + error(...args: any[]) { + this.console.error(...args) + } +} \ No newline at end of file diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index c82d091..a1c2455 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -2,6 +2,7 @@ import { SqlSealDatabase } from "./database"; import { App } from "obsidian"; import { SealObserver } from "./SealObserver"; import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler"; +import { Logger } from "./logger"; export class SqlSeal { public db: SqlSealDatabase @@ -11,7 +12,8 @@ export class SqlSeal { this.db = new SqlSealDatabase(app, verbose) this.observer = new SealObserver(verbose) this.observeAllFileChanges() - this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer) + const logger = new Logger(verbose) + this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger) // FIXME: handle here changes to files and tags? } From a16930a8b446f15c49b779c8d24a45060ca52023 Mon Sep 17 00:00:00 2001 From: Kacper Kula Date: Sun, 1 Sep 2024 20:00:30 +0100 Subject: [PATCH 4/4] chore: bumping up version of the package. --- CHANGELOG.md | 6 ++++++ manifest.json | 2 +- package.json | 2 +- versions.json | 3 ++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37c06fb..77c3c20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 0.6.0 +- fixing CSV loading when names has been converted to camelCase +- Fixing constantly reloading files when modifying sql inside the same block that has table declaration in +- Auto-parsing JSON values with JSON5 +- Better errors when error occurs in select statement + # 0.5.0 - Adding support for JSON objects! Automatically detecting JSON in the frontmatter and converting it to JSON type in SQLite. You can query fields using built-in SQLite functions like `json_extract`, `json_array_length`, etc. More about SQlite functionality [can be found here.](https://www.sqlite.org/json1.html). - Fixed issue with bookean types not being saved properly. diff --git a/manifest.json b/manifest.json index 1846f05..d8fb218 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "sqlseal", "name": "SQLSeal", - "version": "0.5.0", + "version": "0.6.0", "minAppVersion": "0.15.0", "description": "Use SQL in your notes to query your vault files and CSV content.", "author": "hypersphere", diff --git a/package.json b/package.json index 510d318..1f4b3ad 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sqlseal", - "version": "0.5.0", + "version": "0.6.0", "description": "A plugin for Obsidian that allows you to run SQL queries on your notes.", "main": "main.js", "scripts": { diff --git a/versions.json b/versions.json index dc45888..31502ff 100644 --- a/versions.json +++ b/versions.json @@ -4,5 +4,6 @@ "0.3.0": "0.15.0", "0.4.0": "0.15.0", "0.4.1": "0.15.0", - "0.5.0": "0.15.0" + "0.5.0": "0.15.0", + "0.6.0": "0.15.0" } \ No newline at end of file