diff --git a/CHANGELOG.md b/CHANGELOG.md index 580baa0..b0c343b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 0.12.0 +This update does not bring any functionality changes yet but it fixes some problems with underlying architecture, helping to make SQLSeal more efficient in the future. If you work with many datasets, you might see slightly smaller memory usage as the database is now persisted in the IndexedDb (tech comment: as block storage). +Technical update: +Database has been moved to WebWorker, making data loading and retrieval slightly more efficient. Also integrated with [Absurd-SQL](https://github.com/jlongster/absurd-sql) which allows to store SQLite inside IndexedDb block storage, meaning it can be persisted and offloaded from the memory. Due to Obsidian API limitations, we still need to load huge files CSV to the memory instead of streaming their content but once the content is loaded, the memory usage should significantly go down now. More changes using this update to come soon! + # 0.11.0 The biggest update yet with plenty of exciting features: diff --git a/esbuild.config.mjs b/esbuild.config.mjs index b7853b8..0352586 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -10,10 +10,10 @@ if you want to view the source, please visit the github repository of this plugi */ `; +// Plugin to handle WASM files const wasmPlugin = { name: 'wasm', setup(build) { - // Handle direct importing of wasm files build.onResolve({ filter: /\.wasm$/ }, args => { if (args.resolveDir === '') return; return { @@ -22,7 +22,6 @@ const wasmPlugin = { }; }); - // Load and encode the wasm file build.onLoad({ filter: /\.wasm$/, namespace: 'wasm-binary' }, async (args) => { const contents = readFileSync(args.path); const wasmBase64 = contents.toString('base64'); @@ -39,6 +38,78 @@ const wasmPlugin = { }, }; +// Plugin to handle Node.js module shims +// const nodeModulesPlugin = { +// name: 'node-modules', +// setup(build) { +// build.onResolve({ filter: /^path$/ }, args => ({ +// path: args.path, +// namespace: 'node-modules', +// })); + +// build.onLoad({ filter: /^path$/, namespace: 'node-modules' }, () => ({ +// contents: ` +// export function dirname(path) { +// return path.replace(/\\/[^\\/]*$/, ''); +// } +// export function normalize(path) { +// return path; +// } +// `, +// })); + +// build.onResolve({ filter: /^fs$/ }, args => ({ +// path: args.path, +// namespace: 'node-modules', +// })); + +// build.onLoad({ filter: /^fs$/, namespace: 'node-modules' }, () => ({ +// contents: ` +// export function readFileSync() { +// throw new Error('readFileSync is not supported in the browser'); +// } +// `, +// })); +// }, +// }; + +// Plugin to inject worker code +const workerPlugin = { + name: 'worker', + setup(build) { + build.onResolve({ filter: /^virtual:worker-code$/ }, args => ({ + path: args.path, + namespace: 'worker-code', + })); + + build.onLoad({ filter: /.*/, namespace: 'worker-code' }, async () => { + // Build worker code + const result = await esbuild.build({ + entryPoints: ['src/database-worker.ts'], + bundle: true, + write: false, + format: 'iife', + target: 'es2020', + external: ['fs', 'path'], + plugins: [wasmPlugin, /*nodeModulesPlugin*/], + minify: process.argv[2] === 'production', + define: { + 'process.env.NODE_ENV': JSON.stringify(process.argv[2] === 'production' ? 'production' : 'development'), + '__dirname': '"/"' + } + }); + + return { + contents: ` + const workerCode = ${JSON.stringify(result.outputFiles[0].text)}; + export default workerCode; + `, + loader: 'js', + }; + }); + }, +}; + const prod = (process.argv[2] === "production"); const context = await esbuild.context({ @@ -64,14 +135,20 @@ const context = await esbuild.context({ ...builtins ], format: "cjs", - target: "es2018", + target: "ES2020", logLevel: "info", sourcemap: prod ? false : "inline", treeShaking: true, outfile: "main.js", plugins: [ - wasmPlugin - ] + wasmPlugin, + // nodeModulesPlugin, + workerPlugin + ], + define: { + 'process.env.NODE_ENV': JSON.stringify(prod ? 'production' : 'development'), + '__dirname': '"/"' + } }); if (prod) { diff --git a/manifest.json b/manifest.json index 3064b37..0804809 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "sqlseal", "name": "SQLSeal", - "version": "0.11.0", + "version": "0.12.0", "minAppVersion": "0.15.0", "description": "Use SQL in your notes to query your vault files and CSV content.", "author": "hypersphere", diff --git a/package.json b/package.json index 759ec27..0a6cdd6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sqlseal", - "version": "0.11.0", + "version": "0.12.0", "description": "A plugin for Obsidian that allows you to run SQL queries on your notes.", "main": "main.js", "scripts": { @@ -42,13 +42,15 @@ }, "dependencies": { "@ag-grid-community/theming": "^32.3.3", + "@jlongster/sql.js": "^1.6.7", + "absurd-sql": "^0.0.54", "ag-grid-community": "^32.3.3", + "comlink": "^4.4.2", "json5": "^2.2.3", "lodash": "^4.17.21", "markdown-table-ts": "^1.0.3", "node-sql-parser": "^5.3.4", "papaparse": "^5.4.1", - "sql.js": "^1.12.0", "util": "^0.12.5" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd6cb52..821933e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,9 +11,18 @@ importers: '@ag-grid-community/theming': specifier: ^32.3.3 version: 32.3.3 + '@jlongster/sql.js': + specifier: ^1.6.7 + version: 1.6.7 + absurd-sql: + specifier: ^0.0.54 + version: 0.0.54 ag-grid-community: specifier: ^32.3.3 version: 32.3.3 + comlink: + specifier: ^4.4.2 + version: 4.4.2 json5: specifier: ^2.2.3 version: 2.2.3 @@ -29,9 +38,6 @@ importers: papaparse: specifier: ^5.4.1 version: 5.4.1 - sql.js: - specifier: ^1.12.0 - version: 1.12.0 util: specifier: ^0.12.5 version: 0.12.5 @@ -790,6 +796,9 @@ packages: resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jlongster/sql.js@1.6.7': + resolution: {integrity: sha512-4hf0kZr5WPoirdR5hUSfQ9O0JpH/qlW1CaR2wZ6zGrDz1xjSdTPuR8AW/oXzIHnJvZSEvlcIE+dfXJZwh/Lxfw==} + '@jridgewell/gen-mapping@0.3.5': resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} engines: {node: '>=6.0.0'} @@ -1224,6 +1233,9 @@ packages: abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} + absurd-sql@0.0.54: + resolution: {integrity: sha512-p+SWTtpRs2t3sXMLxkTyLRZkEzxTv/zG/Bl93wibegLZTGAHGk68SJMWslRWHBGh63ka/ePGTXGHh1117++45Q==} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1473,6 +1485,9 @@ packages: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} hasBin: true + comlink@4.4.2: + resolution: {integrity: sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -2683,6 +2698,9 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + safari-14-idb-fix@1.0.6: + resolution: {integrity: sha512-oTEQOdMwRX+uCtWCKT1nx2gAeSdpr8elg/2gcaKUH00SJU2xWESfkx11nmXwTRHy7xfQoj1o4TTQvdmuBosTnA==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -2769,9 +2787,6 @@ packages: sprintf-js@1.1.3: resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} - sql.js@1.12.0: - resolution: {integrity: sha512-Bi+43yMx/tUFZVYD4AUscmdL6NHn3gYQ+CM+YheFWLftOmrEC/Mz6Yh7E96Y2WDHYz3COSqT+LP6Z79zgrwJlA==} - ssri@9.0.1: resolution: {integrity: sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} @@ -3792,6 +3807,8 @@ snapshots: '@types/yargs': 17.0.33 chalk: 4.1.2 + '@jlongster/sql.js@1.6.7': {} + '@jridgewell/gen-mapping@0.3.5': dependencies: '@jridgewell/set-array': 1.2.1 @@ -4248,6 +4265,10 @@ snapshots: abbrev@1.1.1: {} + absurd-sql@0.0.54: + dependencies: + safari-14-idb-fix: 1.0.6 + acorn-jsx@5.3.2(acorn@8.14.0): dependencies: acorn: 8.14.0 @@ -4542,6 +4563,8 @@ snapshots: color-support@1.1.3: {} + comlink@4.4.2: {} + comma-separated-tokens@2.0.3: {} concat-map@0.0.1: {} @@ -6017,6 +6040,8 @@ snapshots: dependencies: queue-microtask: 1.2.3 + safari-14-idb-fix@1.0.6: {} + safe-buffer@5.2.1: {} safer-buffer@2.1.2: @@ -6094,8 +6119,6 @@ snapshots: sprintf-js@1.1.3: {} - sql.js@1.12.0: {} - ssri@9.0.1: dependencies: minipass: 3.3.6 diff --git a/src/SqlSealCodeblockHandler.ts b/src/SqlSealCodeblockHandler.ts index cb5e25f..43c2faa 100644 --- a/src/SqlSealCodeblockHandler.ts +++ b/src/SqlSealCodeblockHandler.ts @@ -5,7 +5,6 @@ import { hashString } from "./hash" import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables" import { SqlSealDatabase } from "./database" import { Logger } from "./logger" -import { SyncModel } from "./models/sync" import { TablesManager } from "./dataLoader/collections/tablesManager" import { QueryManager } from "./dataLoader/collections/queryManager" import { parseLanguage, Table, TableWithParentPath } from "./grammar/newParser" @@ -15,7 +14,6 @@ export class SqlSealCodeblockHandler { get globalTables() { return ['files', 'tags', 'tasks'] // Make this come from SealFileSync and plugins. } - syncModel: SyncModel constructor( private readonly app: App, private readonly db: SqlSealDatabase, @@ -24,21 +22,19 @@ export class SqlSealCodeblockHandler { private queryManager: QueryManager, private rendererRegistry: RendererRegistry ) { - this.syncModel = new SyncModel(db) } setupTableSignals(tables: Array) { tables.forEach(t => { - this.logger.log(`Registering table ${t.tableName} -> ${t.fileName}`) this.tableManager.registerTable(t.tableName, t.fileName, t.parentPath) }) } - setupQuerySignals({ statement, tables }: ReturnType, renderer: RenderReturn, ctx: MarkdownPostProcessorContext) { + setupQuerySignals({ statement, tables }: ReturnType, renderer: RenderReturn, ctx: MarkdownPostProcessorContext, el: Element) { const frontmatter = resolveFrontmatter(ctx, this.app) const renderSelect = async () => { try { - const { data, columns } = this.db.select(statement, frontmatter ?? {}) + const { data, columns } = await this.db.select(statement, frontmatter ?? {}) renderer.render({ data, columns }) } catch (e) { renderer.error(e.toString()) @@ -47,7 +43,11 @@ export class SqlSealCodeblockHandler { const sig = this.queryManager.registerQuery(ctx.docId, tables) - sig(() => { + const unsubscribe = sig(() => { + if (!el.isConnected) { + unsubscribe() + return + } renderSelect() }) } @@ -88,7 +88,7 @@ export class SqlSealCodeblockHandler { try { if (results.queryPart) { const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix) - this.setupQuerySignals({ statement, tables }, renderer!, ctx) + this.setupQuerySignals({ statement, tables }, renderer!, ctx, el) } } catch (e) { renderer!.error(e.toString()) diff --git a/src/dataLoader/collections/filesManager.ts b/src/dataLoader/collections/filesManager.ts index 94e0f97..2a75b01 100644 --- a/src/dataLoader/collections/filesManager.ts +++ b/src/dataLoader/collections/filesManager.ts @@ -1,14 +1,18 @@ import { Signal, SignalUnsubscriber } from "src/utils/signal"; import { dataTransformer, DataTransformerOut } from "../dataTransformer"; -import { csvFileSignal } from "../csvFile"; +import { CSVData, csvFileSignal } from "../csvFile"; import { App, TFile, Vault } from "obsidian"; import { isNull } from "lodash"; +import { SqlSealDatabase } from "src/database"; +import { SyncModel } from "src/models/sync"; export class FilesManager { files: Map> = new Map() - inputFiles: Map> = new Map() + inputFiles: Map> = new Map() unregisters: Array = [] - constructor(private vault: Vault, private app: App) { + syncModel: SyncModel + constructor(private vault: Vault, private app: App, private db: SqlSealDatabase) { + this.syncModel = new SyncModel(this.db) this.vault.on('modify', async (file) => { if (this.files.has(file.path)) { this.inputFiles.get(file.path)!(await this.loadFile(file.path)) @@ -30,10 +34,19 @@ export class FilesManager { return data } - getFile(url: string, sourcePath: string) { + getFile(url: string, sourcePath: string = '/') { return this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) } + getFileHandler(url: string, sourcePath?: string) { + let file: TFile | null; + if (sourcePath) { + file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) + } else { + file = this.vault.getFileByPath(url) + } + } + doesFileExist(url: string, sourcePath: string) { const file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath) return !isNull(file) @@ -43,6 +56,10 @@ export class FilesManager { this.unregisters.forEach(u => u()) } + private saveManagedRecord(filename: string, checksum: string) { + + } + getFileSignal(filename: string): Signal { if (this.files.has(filename)) { return this.files.get(filename)! @@ -50,12 +67,13 @@ export class FilesManager { const fileSignal = csvFileSignal(filename) const signal = dataTransformer(fileSignal) + // Check if it's already in the database, if so, we can just load it again. + this.db + // FIXME: add here ability to watch changes. this.files.set(filename, signal) this.inputFiles.set(filename, fileSignal) - - // now firing signal for the file this.loadFile(filename).then(fileSignal) return signal diff --git a/src/dataLoader/collections/tablesManager.ts b/src/dataLoader/collections/tablesManager.ts index 6536876..265ac55 100644 --- a/src/dataLoader/collections/tablesManager.ts +++ b/src/dataLoader/collections/tablesManager.ts @@ -26,7 +26,7 @@ export class TablesManager { if (this.tableLinks.has(tableName)) { const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)! - if (prevFileName === fileName) { + if (prevFileName === file.path) { return } unlink() diff --git a/src/dataLoader/csvFile.ts b/src/dataLoader/csvFile.ts index fa00ed6..c8000d5 100644 --- a/src/dataLoader/csvFile.ts +++ b/src/dataLoader/csvFile.ts @@ -1,3 +1,4 @@ +import { TFile } from "obsidian"; import { createSignal, Signal } from "src/utils/signal"; export type CSVData = string diff --git a/src/dataLoader/dataTransformer.ts b/src/dataLoader/dataTransformer.ts index a204625..0d367ff 100644 --- a/src/dataLoader/dataTransformer.ts +++ b/src/dataLoader/dataTransformer.ts @@ -3,6 +3,7 @@ import { CSVData } from "./csvFile" import { parse } from 'papaparse' import { toTypeStatements } from "../utils" import { sanitise } from "../utils/sanitiseColumn" +import { TFile } from "obsidian" export const parseData = (csvData: string) => { @@ -17,11 +18,9 @@ export const parseData = (csvData: string) => { export const dataTransformer = (s: Signal) => { - const sig = derivedSignal([s], (csvData) => { - // FIXME: fix header here. + const sig = derivedSignal([s], (data) => { try { - const parsed = parseData(csvData) - + const parsed = parseData(data) const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data) return typeStatements } catch (e) { diff --git a/src/dataLoader/tableSignal.ts b/src/dataLoader/tableSignal.ts index 599b979..e3641d9 100644 --- a/src/dataLoader/tableSignal.ts +++ b/src/dataLoader/tableSignal.ts @@ -4,7 +4,7 @@ import { SqlSealDatabase } from "src/database"; import { FieldTypes } from "src/utils"; export const linkTableWithFile = (dataSig: Signal, tableSignal: Signal, tableName: string, db: SqlSealDatabase) => { - return dataSig(({ data, types}) => { + return dataSig(({ data, types }) => { // Check if the columns are exactly the same. If not, delete and reinstantiate the table. diff --git a/src/database-worker.ts b/src/database-worker.ts new file mode 100644 index 0000000..e277ab4 --- /dev/null +++ b/src/database-worker.ts @@ -0,0 +1,219 @@ +import * as Comlink from "comlink" +import initSqlJs from '@jlongster/sql.js'; +import wasmBinary from '../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm' +import { SQLiteFS } from 'absurd-sql'; +import IndexedDBBackend from '../node_modules/absurd-sql/dist/indexeddb-backend.js'; +import type { BindParams, Database, Statement } from "sql.js"; +import { sanitise } from "./utils/sanitiseColumn"; +import { FieldTypes } from "./utils"; + + +function toObjectArray(stmt: Statement) { + const ret = [] + while (stmt.step()) { + ret.push(stmt.getAsObject()) + } + return ret +} + +function recordToBindParams(record: Record) { + const bindParams = Object.fromEntries(Object.entries(record).map(([key, val]) => ([`@${key}`, val]))) as BindParams + return bindParams +} + +const formatData = (data: Record) => { + return Object.keys(data).reduce((ret, key) => { + if (typeof data[key] === 'boolean') { + return { + ...ret, + [key]: data[key] ? 1 : 0 + } + } + if (!data[key]) { + return { + ...ret, + [key]: null + } + } + if (typeof data[key] === 'object' || Array.isArray(data[key])) { + return { + ...ret, + [key]: JSON.stringify(data[key]) + } + } + return { + ...ret, + [key]: data[key] + } + }, {}) +} + +// Disabling SharedArrayBuffer - otherwise Absurd-SQL doesn't handle properly it. +self.SharedArrayBuffer = undefined as any + +export class WorkerDatabase { + + private db: Database + + constructor() { + + } + + defineCustomFunctions() { + this.db.create_function('a', (href: string, name: string) => { + const linkObject = { + type: 'link', + href: href, + name: name || href + }; + return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`; + }); + + this.db.create_function('a', (href: string) => { + const linkObject = { + type: 'link', + href: href, + name: href + }; + return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`; + }); + + this.db.create_function('img', (href: string) => { + const imgObject = { + type: 'img', + href: href + } + return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})` + }) + + this.db.create_function('img', (href: string, path: string) => { + const imgObject = { + type: 'img', + path, + href + } + return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})` + }) + + this.db.create_function('checkbox', (val: string) => { + const imgObject = { + type: 'checkbox', + value: val + } + return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})` + }) + } + + createTableText(tableName: string, fields: string[]) { + const fieldsString = fields.map(f => `${f} TEXT`).join(',') + this.db.exec(`CREATE TABLE IF NOT EXISTS ${tableName} (:fieldsString)`, { + tableName, + fieldsString + }) + } + + async createTable(tableName: string, fields: Record) { + const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type]) + const uniqueFields = [...new Map(transformedFiels.map(item => + [item[0], item])).values()] + const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`) + // FIXME: probably use schema generator, for now create with hardcoded fields + await this.dropTable(tableName) + const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} ( + ${sqlFields.join(', ')} + );` + + this.db.prepare(createSQL).run() + await this.clearTable(tableName) + + // Dropping data. + } + + async clearTable(tableName: string) { + this.db.prepare(`DELETE FROM ${tableName}`).run() + + } + + async insertData(tableName: string, data: Record[]) { + data.forEach(d => { + const columns = Object.keys(d) + const insertStatement = this.db.prepare(`INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`); + insertStatement.run(recordToBindParams(formatData(d))) + }) + } + + async dropTable(tableName: string) { + this.db.run(`DROP TABLE IF EXISTS ${tableName}`) + } + + async select(query: string, params: Record) { + const stmt = this.db.prepare(query, recordToBindParams(params)) + return { + data: toObjectArray(stmt), + columns: stmt.getColumnNames() + } + } + + async updateData(tableName: string, data: Array>) { + const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); + data.forEach((d: Record) => { + const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`) + stmt.run(recordToBindParams(d)) + }) + } + + async deleteData(name: string, data: Array>, key: string = 'id') { + data.forEach(d => { + const stmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`); + stmt.run({ + [`@${key}`]: d[key] + } as BindParams) + }) + } + + async connect() { + try { + const SQL = await initSqlJs({ + wasmBinary: wasmBinary + }); + + let sqlFS = new SQLiteFS(SQL.FS, new IndexedDBBackend(() => { + console.error('unable to write to indexedDb') + })); + SQL.register_for_idb(sqlFS); + + SQL.FS.mkdir('/sql'); + SQL.FS.mount(sqlFS, {}, '/sql'); + + const path = 'sql/sqlseal.sqlite'; + + let stream = SQL.FS.open(path, 'a+'); + await stream.node.contents.readIfFallback(); + SQL.FS.close(stream); + + const db = new SQL.Database('sql/sqlseal.sqlite', { filename: true }); + + let cacheSize = 0; + let pageSize = 4096; + + db.exec(` + PRAGMA cache_size=-${cacheSize}; + PRAGMA journal_mode=MEMORY; + PRAGMA page_size=${pageSize}; + VACUUM; + `); + this.db = db + this.defineCustomFunctions() + return + } catch (e) { + console.error('Error while setting up database', e); + throw e; + } + } + + async disconnect() { + // FIXME: implement. + } +} + +Comlink.expose(WorkerDatabase); \ No newline at end of file diff --git a/src/database.ts b/src/database.ts index 9116f30..1893e44 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,14 +1,12 @@ import { App } from "obsidian" -import path from 'path' -import Papa from 'papaparse' -import { prefixedIfNotGlobal } from "./sqlReparseTables" -import { camelCase } from 'lodash' -import { dataToCamelCase, fetchBlobData, FieldTypes, predictJson, predictType, toTypeStatements } from "./utils" -import os from 'os' -import fs from 'fs' +import { FieldTypes, toTypeStatements } from "./utils" import { sanitise } from "./utils/sanitiseColumn" import initSqlJs, { BindParams, Database, Statement } from 'sql.js' import wasmBinary from '../node_modules/sql.js/dist/sql-wasm.wasm' +import * as Comlink from 'comlink' +import workerCode from 'virtual:worker-code' +import { WorkerDatabase } from "./database-worker" + export interface FieldDefinition { name: string; @@ -43,12 +41,9 @@ const formatData = (data: Record) => { } export class SqlSealDatabase { - private savedDatabases: Record = {} - db: Database - private SQL: initSqlJs.SqlJsStatic + db: Comlink.Remote private isConnected = false private connectingPromise: Promise; - private connectingPromiseResolve: (value: void | PromiseLike) => void constructor(private readonly app: App, private readonly verbose = false) { } @@ -62,76 +57,34 @@ export class SqlSealDatabase { return this.connectingPromise } - this.connectingPromise = new Promise((resolve) => { - this.connectingPromiseResolve = resolve - }) + this.connectingPromise = new Promise(async (resolve, reject) => { + try { + const blob = new Blob([workerCode], { type: 'text/javascript' }); + const workerUrl = URL.createObjectURL(blob); + + const worker = new Worker(workerUrl, { + name: 'SQLSeal Database' + }); + const DatabaseWrap = Comlink.wrap(worker) - try { - this.SQL = await initSqlJs({ - wasmBinary: wasmBinary - }) - - this.db = new this.SQL.Database() - await this.defineCustomFunctions() - - this.isConnected = true - this.connectingPromiseResolve() - } catch (e) { - console.error('Error initializing SQLite database:', e) - } - } + const instance = await new DatabaseWrap() - private async defineCustomFunctions() { - this.db.create_function('a', (href: string, name: string) => { - const linkObject = { - type: 'link', - href: href, - name: name || href - }; - return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`; - }); - - this.db.create_function('a', (href: string) => { - const linkObject = { - type: 'link', - href: href, - name: href - }; - return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`; - }); - - this.db.create_function('img', (href: string) => { - const imgObject = { - type: 'img', - href: href + await instance.connect() + this.db = instance + this.isConnected = true + resolve() + } catch (e) { + reject(e) } - return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})` }) - - this.db.create_function('img', (href: string, path: string) => { - const imgObject = { - type: 'img', - path, - href - } - return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})` - }) - - this.db.create_function('checkbox', (val: string) => { - const imgObject = { - type: 'checkbox', - value: val - } - return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})` - }) - + return this.connectingPromise } async disconect() { if (!this.isConnected) { return } - this.db.close() + this.db.disconnect() this.isConnected = false } @@ -143,88 +96,47 @@ export class SqlSealDatabase { return schema } - async addNewColumns(name: string, data: Array>) { - const schema = await this.getSchema(data) - const currentSchema = this.toObjectsArray(this.db.prepare(`PRAGMA table_info(${name})`)) - const currentFields = currentSchema.map((f: any) => f.name) - const newFields = Object.keys(schema).filter(f => !currentFields.includes(f)) + // async addNewColumns(name: string, data: Array>) { + // const schema = await this.getSchema(data) + // const currentSchema = this.toObjectsArray(this.db.prepare(`PRAGMA table_info(${name})`)) + // const currentFields = currentSchema.map((f: any) => f.name) + // const newFields = Object.keys(schema).filter(f => !currentFields.includes(f)) - if (newFields.length === 0) { - return - } + // if (newFields.length === 0) { + // return + // } - const alter = this.db.prepare(`ALTER TABLE ${name} ADD - COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`) - alter.run() + // const alter = this.db.prepare(`ALTER TABLE ${name} ADD + // COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`) + // alter.run() + // } + + async updateData(name: string, data: Array>) { + return this.db.updateData(name, data) } - private toObjectsArray(stmt: Statement) { - const ret = [] - while(stmt.step()) { - ret.push(stmt.getAsObject()) - } - return ret - } - - private recordToBindParams(record: Record) { - const bindParams = Object.fromEntries(Object.entries(record).map(([key, val]) => ([`@${key}`, val]))) as BindParams - return bindParams - } - - updateData(name: string, data: Array>) { - const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); - const update = this.db.prepare(`UPDATE ${name} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`); - data.forEach((d: Record) => { - const stmt = this.db.prepare(`UPDATE ${name} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`) - stmt.run(this.recordToBindParams(d)) - }) - } - - deleteData(name: string, data: Array>, key: string = 'id') { - data.forEach(d => { - const stmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`); - stmt.run({ - [`@${key}`]: d[key] - } as BindParams) - }) + async deleteData(name: string, data: Array>, key: string = 'id') { + return this.db.deleteData(name, data, key) } async insertData(name: string, inData: Array>) { - inData.forEach(d => { - const columns = Object.keys(d) - const insertStatement = this.db.prepare(`INSERT INTO ${name} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`); - insertStatement.run(this.recordToBindParams(formatData(d))) - }) + return this.db.insertData(name, inData) } - dropTable(name: string) { - this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run() - this.savedDatabases[name] = false + async dropTable(name: string) { + return this.db.dropTable(name) } - createTableClean(name: string, fields: Array) { - const sqlFields = fields.map(({ name, type }) => `${name} ${type}`).join(', ') - const createSql = `CREATE TABLE IF NOT EXISTS ${name} (${sqlFields})` - this.db.prepare(createSql).run() - this.savedDatabases[name] = true + async createTableClean(name: string, fields: Array) { + const fieldsToRecord = fields.reduce((acc, f) => ({ + ...acc, + [f.name]: f.type + }), {} as Record) as Record + await this.createTable(name, fieldsToRecord) } async createTable(name: string, fields: Record) { - const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type]) - const uniqueFields = [...new Map(transformedFiels.map(item => - [item[0], item])).values()] - const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`) - // FIXME: probably use schema generator, for now create with hardcoded fields - this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run() - const createSQL = `CREATE TABLE IF NOT EXISTS ${name} ( - ${sqlFields.join(', ')} - );` - - this.db.prepare(createSQL).run() - this.savedDatabases[name] = true - - // Dropping data. - this.db.prepare(`DELETE FROM ${name}`).run() + await this.db.createTable(name, fields) } async getSchema(data: Array>) { const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); @@ -232,12 +144,8 @@ export class SqlSealDatabase { return types; } - select(statement: string, frontmatter: Record) { - const stmt = this.db.prepare(statement) - stmt.bind(this.recordToBindParams(frontmatter ?? {})) - return { - data: this.toObjectsArray(stmt) as Record[], - columns: stmt.getColumnNames() - } + async select(statement: string, frontmatter: Record) { + const result = await this.db.select(statement, frontmatter) + return result } } \ No newline at end of file diff --git a/src/models/sync.ts b/src/models/sync.ts index 46c4662..14c25ca 100644 --- a/src/models/sync.ts +++ b/src/models/sync.ts @@ -1,5 +1,6 @@ import { SqlSealDatabase } from "src/database"; + export interface SyncEntry { filename: string; url: string; @@ -20,40 +21,34 @@ export class SyncModel { createTableIfNotExists() { this.db.createTable(this.tableName, { 'filename': 'TEXT', - 'url': 'TEXT', - 'name': 'TEXT', + 'checksum': 'TEXT', 'syncedAt': 'TEXT', - 'metadata': 'JSON' }) } - registerSync(filename: string, url: string, tableName: string) { + registerSync(filename: string, checksum: string) { this.db.insertData(this.tableName, [{ filename, - url, - name: tableName, + checksum, syncedAt: Date.now(), - metadata: { } }]) } - getSync(filename: string, tableName: string) { - const { data } = this.db.select(`SELECT * + async getSync(filename: string) { + const { data } = await this.db.select(`SELECT * FROM sqlseal_sync - WHERE filename = @filename - AND name = @name`, + WHERE filename = @filename`, { - name: tableName, filename: filename }) if (data) { - return data[0] as SyncEntry + return data[0] as unknown as SyncEntry } return null } - removeSync(filename: string, tableName: string) { + removeSync(filename: string) { // FIXME: do not use exposed db, instead implement "exec" method inside the file. this.db.db.prepare('DELETE FROM sqlseal_sync WHERE filename = :filename AND name = :name').run({ filename: filename, diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index bd232b7..247bc69 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -15,7 +15,7 @@ export class SqlSeal { this.db = new SqlSealDatabase(app, verbose) const logger = new Logger(verbose) - const fileManager = new FilesManager(this.app.vault, this.app) + const fileManager = new FilesManager(this.app.vault, this.app, this.db) this.tablesManager = new TablesManager(fileManager, this.db) this.tablesManager.getTableSignal('files') this.tablesManager.getTableSignal('tags') diff --git a/versions.json b/versions.json index 59cca82..785c68e 100644 --- a/versions.json +++ b/versions.json @@ -13,5 +13,6 @@ "0.9.2": "0.15.0", "0.10.0": "0.15.0", "0.10.1": "0.15.0", - "0.11.0": "0.15.0" + "0.11.0": "0.15.0", + "0.12.0": "0.15.0" } \ No newline at end of file