mirror of
https://github.com/h-sphere/sql-seal.git
synced 2026-07-22 10:10:28 +00:00
Merge pull request #21 from h-sphere/feat/sql-worker
feat: WebWorker + Absurd-SQL
This commit is contained in:
Kacper Kula
GitHub
commit
fc490cf7f2
16 changed files with 446 additions and 198 deletions
|
|
@ -1,3 +1,8 @@
|
|||
# 0.12.0
|
||||
This update does not bring any functionality changes yet but it fixes some problems with underlying architecture, helping to make SQLSeal more efficient in the future. If you work with many datasets, you might see slightly smaller memory usage as the database is now persisted in the IndexedDb (tech comment: as block storage).
|
||||
Technical update:
|
||||
Database has been moved to WebWorker, making data loading and retrieval slightly more efficient. Also integrated with [Absurd-SQL](https://github.com/jlongster/absurd-sql) which allows to store SQLite inside IndexedDb block storage, meaning it can be persisted and offloaded from the memory. Due to Obsidian API limitations, we still need to load huge files CSV to the memory instead of streaming their content but once the content is loaded, the memory usage should significantly go down now. More changes using this update to come soon!
|
||||
|
||||
# 0.11.0
|
||||
The biggest update yet with plenty of exciting features:
|
||||
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ if you want to view the source, please visit the github repository of this plugi
|
|||
*/
|
||||
`;
|
||||
|
||||
// Plugin to handle WASM files
|
||||
const wasmPlugin = {
|
||||
name: 'wasm',
|
||||
setup(build) {
|
||||
// Handle direct importing of wasm files
|
||||
build.onResolve({ filter: /\.wasm$/ }, args => {
|
||||
if (args.resolveDir === '') return;
|
||||
return {
|
||||
|
|
@ -22,7 +22,6 @@ const wasmPlugin = {
|
|||
};
|
||||
});
|
||||
|
||||
// Load and encode the wasm file
|
||||
build.onLoad({ filter: /\.wasm$/, namespace: 'wasm-binary' }, async (args) => {
|
||||
const contents = readFileSync(args.path);
|
||||
const wasmBase64 = contents.toString('base64');
|
||||
|
|
@ -39,6 +38,78 @@ const wasmPlugin = {
|
|||
},
|
||||
};
|
||||
|
||||
// Plugin to handle Node.js module shims
|
||||
// const nodeModulesPlugin = {
|
||||
// name: 'node-modules',
|
||||
// setup(build) {
|
||||
// build.onResolve({ filter: /^path$/ }, args => ({
|
||||
// path: args.path,
|
||||
// namespace: 'node-modules',
|
||||
// }));
|
||||
|
||||
// build.onLoad({ filter: /^path$/, namespace: 'node-modules' }, () => ({
|
||||
// contents: `
|
||||
// export function dirname(path) {
|
||||
// return path.replace(/\\/[^\\/]*$/, '');
|
||||
// }
|
||||
// export function normalize(path) {
|
||||
// return path;
|
||||
// }
|
||||
// `,
|
||||
// }));
|
||||
|
||||
// build.onResolve({ filter: /^fs$/ }, args => ({
|
||||
// path: args.path,
|
||||
// namespace: 'node-modules',
|
||||
// }));
|
||||
|
||||
// build.onLoad({ filter: /^fs$/, namespace: 'node-modules' }, () => ({
|
||||
// contents: `
|
||||
// export function readFileSync() {
|
||||
// throw new Error('readFileSync is not supported in the browser');
|
||||
// }
|
||||
// `,
|
||||
// }));
|
||||
// },
|
||||
// };
|
||||
|
||||
// Plugin to inject worker code
|
||||
const workerPlugin = {
|
||||
name: 'worker',
|
||||
setup(build) {
|
||||
build.onResolve({ filter: /^virtual:worker-code$/ }, args => ({
|
||||
path: args.path,
|
||||
namespace: 'worker-code',
|
||||
}));
|
||||
|
||||
build.onLoad({ filter: /.*/, namespace: 'worker-code' }, async () => {
|
||||
// Build worker code
|
||||
const result = await esbuild.build({
|
||||
entryPoints: ['src/database-worker.ts'],
|
||||
bundle: true,
|
||||
write: false,
|
||||
format: 'iife',
|
||||
target: 'es2020',
|
||||
external: ['fs', 'path'],
|
||||
plugins: [wasmPlugin, /*nodeModulesPlugin*/],
|
||||
minify: process.argv[2] === 'production',
|
||||
define: {
|
||||
'process.env.NODE_ENV': JSON.stringify(process.argv[2] === 'production' ? 'production' : 'development'),
|
||||
'__dirname': '"/"'
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
contents: `
|
||||
const workerCode = ${JSON.stringify(result.outputFiles[0].text)};
|
||||
export default workerCode;
|
||||
`,
|
||||
loader: 'js',
|
||||
};
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
const prod = (process.argv[2] === "production");
|
||||
|
||||
const context = await esbuild.context({
|
||||
|
|
@ -64,14 +135,20 @@ const context = await esbuild.context({
|
|||
...builtins
|
||||
],
|
||||
format: "cjs",
|
||||
target: "es2018",
|
||||
target: "ES2020",
|
||||
logLevel: "info",
|
||||
sourcemap: prod ? false : "inline",
|
||||
treeShaking: true,
|
||||
outfile: "main.js",
|
||||
plugins: [
|
||||
wasmPlugin
|
||||
]
|
||||
wasmPlugin,
|
||||
// nodeModulesPlugin,
|
||||
workerPlugin
|
||||
],
|
||||
define: {
|
||||
'process.env.NODE_ENV': JSON.stringify(prod ? 'production' : 'development'),
|
||||
'__dirname': '"/"'
|
||||
}
|
||||
});
|
||||
|
||||
if (prod) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"id": "sqlseal",
|
||||
"name": "SQLSeal",
|
||||
"version": "0.11.0",
|
||||
"version": "0.12.0",
|
||||
"minAppVersion": "0.15.0",
|
||||
"description": "Use SQL in your notes to query your vault files and CSV content.",
|
||||
"author": "hypersphere",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "sqlseal",
|
||||
"version": "0.11.0",
|
||||
"version": "0.12.0",
|
||||
"description": "A plugin for Obsidian that allows you to run SQL queries on your notes.",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
|
|
@ -42,13 +42,15 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@ag-grid-community/theming": "^32.3.3",
|
||||
"@jlongster/sql.js": "^1.6.7",
|
||||
"absurd-sql": "^0.0.54",
|
||||
"ag-grid-community": "^32.3.3",
|
||||
"comlink": "^4.4.2",
|
||||
"json5": "^2.2.3",
|
||||
"lodash": "^4.17.21",
|
||||
"markdown-table-ts": "^1.0.3",
|
||||
"node-sql-parser": "^5.3.4",
|
||||
"papaparse": "^5.4.1",
|
||||
"sql.js": "^1.12.0",
|
||||
"util": "^0.12.5"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,18 @@ importers:
|
|||
'@ag-grid-community/theming':
|
||||
specifier: ^32.3.3
|
||||
version: 32.3.3
|
||||
'@jlongster/sql.js':
|
||||
specifier: ^1.6.7
|
||||
version: 1.6.7
|
||||
absurd-sql:
|
||||
specifier: ^0.0.54
|
||||
version: 0.0.54
|
||||
ag-grid-community:
|
||||
specifier: ^32.3.3
|
||||
version: 32.3.3
|
||||
comlink:
|
||||
specifier: ^4.4.2
|
||||
version: 4.4.2
|
||||
json5:
|
||||
specifier: ^2.2.3
|
||||
version: 2.2.3
|
||||
|
|
@ -29,9 +38,6 @@ importers:
|
|||
papaparse:
|
||||
specifier: ^5.4.1
|
||||
version: 5.4.1
|
||||
sql.js:
|
||||
specifier: ^1.12.0
|
||||
version: 1.12.0
|
||||
util:
|
||||
specifier: ^0.12.5
|
||||
version: 0.12.5
|
||||
|
|
@ -790,6 +796,9 @@ packages:
|
|||
resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==}
|
||||
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
|
||||
|
||||
'@jlongster/sql.js@1.6.7':
|
||||
resolution: {integrity: sha512-4hf0kZr5WPoirdR5hUSfQ9O0JpH/qlW1CaR2wZ6zGrDz1xjSdTPuR8AW/oXzIHnJvZSEvlcIE+dfXJZwh/Lxfw==}
|
||||
|
||||
'@jridgewell/gen-mapping@0.3.5':
|
||||
resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==}
|
||||
engines: {node: '>=6.0.0'}
|
||||
|
|
@ -1224,6 +1233,9 @@ packages:
|
|||
abbrev@1.1.1:
|
||||
resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
|
||||
|
||||
absurd-sql@0.0.54:
|
||||
resolution: {integrity: sha512-p+SWTtpRs2t3sXMLxkTyLRZkEzxTv/zG/Bl93wibegLZTGAHGk68SJMWslRWHBGh63ka/ePGTXGHh1117++45Q==}
|
||||
|
||||
acorn-jsx@5.3.2:
|
||||
resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
|
||||
peerDependencies:
|
||||
|
|
@ -1473,6 +1485,9 @@ packages:
|
|||
resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==}
|
||||
hasBin: true
|
||||
|
||||
comlink@4.4.2:
|
||||
resolution: {integrity: sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==}
|
||||
|
||||
comma-separated-tokens@2.0.3:
|
||||
resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==}
|
||||
|
||||
|
|
@ -2683,6 +2698,9 @@ packages:
|
|||
run-parallel@1.2.0:
|
||||
resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
|
||||
|
||||
safari-14-idb-fix@1.0.6:
|
||||
resolution: {integrity: sha512-oTEQOdMwRX+uCtWCKT1nx2gAeSdpr8elg/2gcaKUH00SJU2xWESfkx11nmXwTRHy7xfQoj1o4TTQvdmuBosTnA==}
|
||||
|
||||
safe-buffer@5.2.1:
|
||||
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
|
||||
|
||||
|
|
@ -2769,9 +2787,6 @@ packages:
|
|||
sprintf-js@1.1.3:
|
||||
resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==}
|
||||
|
||||
sql.js@1.12.0:
|
||||
resolution: {integrity: sha512-Bi+43yMx/tUFZVYD4AUscmdL6NHn3gYQ+CM+YheFWLftOmrEC/Mz6Yh7E96Y2WDHYz3COSqT+LP6Z79zgrwJlA==}
|
||||
|
||||
ssri@9.0.1:
|
||||
resolution: {integrity: sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==}
|
||||
engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0}
|
||||
|
|
@ -3792,6 +3807,8 @@ snapshots:
|
|||
'@types/yargs': 17.0.33
|
||||
chalk: 4.1.2
|
||||
|
||||
'@jlongster/sql.js@1.6.7': {}
|
||||
|
||||
'@jridgewell/gen-mapping@0.3.5':
|
||||
dependencies:
|
||||
'@jridgewell/set-array': 1.2.1
|
||||
|
|
@ -4248,6 +4265,10 @@ snapshots:
|
|||
|
||||
abbrev@1.1.1: {}
|
||||
|
||||
absurd-sql@0.0.54:
|
||||
dependencies:
|
||||
safari-14-idb-fix: 1.0.6
|
||||
|
||||
acorn-jsx@5.3.2(acorn@8.14.0):
|
||||
dependencies:
|
||||
acorn: 8.14.0
|
||||
|
|
@ -4542,6 +4563,8 @@ snapshots:
|
|||
|
||||
color-support@1.1.3: {}
|
||||
|
||||
comlink@4.4.2: {}
|
||||
|
||||
comma-separated-tokens@2.0.3: {}
|
||||
|
||||
concat-map@0.0.1: {}
|
||||
|
|
@ -6017,6 +6040,8 @@ snapshots:
|
|||
dependencies:
|
||||
queue-microtask: 1.2.3
|
||||
|
||||
safari-14-idb-fix@1.0.6: {}
|
||||
|
||||
safe-buffer@5.2.1: {}
|
||||
|
||||
safer-buffer@2.1.2:
|
||||
|
|
@ -6094,8 +6119,6 @@ snapshots:
|
|||
|
||||
sprintf-js@1.1.3: {}
|
||||
|
||||
sql.js@1.12.0: {}
|
||||
|
||||
ssri@9.0.1:
|
||||
dependencies:
|
||||
minipass: 3.3.6
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import { hashString } from "./hash"
|
|||
import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables"
|
||||
import { SqlSealDatabase } from "./database"
|
||||
import { Logger } from "./logger"
|
||||
import { SyncModel } from "./models/sync"
|
||||
import { TablesManager } from "./dataLoader/collections/tablesManager"
|
||||
import { QueryManager } from "./dataLoader/collections/queryManager"
|
||||
import { parseLanguage, Table, TableWithParentPath } from "./grammar/newParser"
|
||||
|
|
@ -15,7 +14,6 @@ export class SqlSealCodeblockHandler {
|
|||
get globalTables() {
|
||||
return ['files', 'tags', 'tasks'] // Make this come from SealFileSync and plugins.
|
||||
}
|
||||
syncModel: SyncModel
|
||||
constructor(
|
||||
private readonly app: App,
|
||||
private readonly db: SqlSealDatabase,
|
||||
|
|
@ -24,21 +22,19 @@ export class SqlSealCodeblockHandler {
|
|||
private queryManager: QueryManager,
|
||||
private rendererRegistry: RendererRegistry
|
||||
) {
|
||||
this.syncModel = new SyncModel(db)
|
||||
}
|
||||
|
||||
setupTableSignals(tables: Array<TableWithParentPath>) {
|
||||
tables.forEach(t => {
|
||||
this.logger.log(`Registering table ${t.tableName} -> ${t.fileName}`)
|
||||
this.tableManager.registerTable(t.tableName, t.fileName, t.parentPath)
|
||||
})
|
||||
}
|
||||
|
||||
setupQuerySignals({ statement, tables }: ReturnType<typeof updateTables>, renderer: RenderReturn, ctx: MarkdownPostProcessorContext) {
|
||||
setupQuerySignals({ statement, tables }: ReturnType<typeof updateTables>, renderer: RenderReturn, ctx: MarkdownPostProcessorContext, el: Element) {
|
||||
const frontmatter = resolveFrontmatter(ctx, this.app)
|
||||
const renderSelect = async () => {
|
||||
try {
|
||||
const { data, columns } = this.db.select(statement, frontmatter ?? {})
|
||||
const { data, columns } = await this.db.select(statement, frontmatter ?? {})
|
||||
renderer.render({ data, columns })
|
||||
} catch (e) {
|
||||
renderer.error(e.toString())
|
||||
|
|
@ -47,7 +43,11 @@ export class SqlSealCodeblockHandler {
|
|||
|
||||
|
||||
const sig = this.queryManager.registerQuery(ctx.docId, tables)
|
||||
sig(() => {
|
||||
const unsubscribe = sig(() => {
|
||||
if (!el.isConnected) {
|
||||
unsubscribe()
|
||||
return
|
||||
}
|
||||
renderSelect()
|
||||
})
|
||||
}
|
||||
|
|
@ -88,7 +88,7 @@ export class SqlSealCodeblockHandler {
|
|||
try {
|
||||
if (results.queryPart) {
|
||||
const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix)
|
||||
this.setupQuerySignals({ statement, tables }, renderer!, ctx)
|
||||
this.setupQuerySignals({ statement, tables }, renderer!, ctx, el)
|
||||
}
|
||||
} catch (e) {
|
||||
renderer!.error(e.toString())
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
import { Signal, SignalUnsubscriber } from "src/utils/signal";
|
||||
import { dataTransformer, DataTransformerOut } from "../dataTransformer";
|
||||
import { csvFileSignal } from "../csvFile";
|
||||
import { CSVData, csvFileSignal } from "../csvFile";
|
||||
import { App, TFile, Vault } from "obsidian";
|
||||
import { isNull } from "lodash";
|
||||
import { SqlSealDatabase } from "src/database";
|
||||
import { SyncModel } from "src/models/sync";
|
||||
|
||||
export class FilesManager {
|
||||
files: Map<string, Signal<DataTransformerOut>> = new Map()
|
||||
inputFiles: Map<string, Signal<string>> = new Map()
|
||||
inputFiles: Map<string, Signal<CSVData>> = new Map()
|
||||
unregisters: Array<SignalUnsubscriber> = []
|
||||
constructor(private vault: Vault, private app: App) {
|
||||
syncModel: SyncModel
|
||||
constructor(private vault: Vault, private app: App, private db: SqlSealDatabase) {
|
||||
this.syncModel = new SyncModel(this.db)
|
||||
this.vault.on('modify', async (file) => {
|
||||
if (this.files.has(file.path)) {
|
||||
this.inputFiles.get(file.path)!(await this.loadFile(file.path))
|
||||
|
|
@ -30,10 +34,19 @@ export class FilesManager {
|
|||
return data
|
||||
}
|
||||
|
||||
getFile(url: string, sourcePath: string) {
|
||||
getFile(url: string, sourcePath: string = '/') {
|
||||
return this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
}
|
||||
|
||||
getFileHandler(url: string, sourcePath?: string) {
|
||||
let file: TFile | null;
|
||||
if (sourcePath) {
|
||||
file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
} else {
|
||||
file = this.vault.getFileByPath(url)
|
||||
}
|
||||
}
|
||||
|
||||
doesFileExist(url: string, sourcePath: string) {
|
||||
const file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
return !isNull(file)
|
||||
|
|
@ -43,6 +56,10 @@ export class FilesManager {
|
|||
this.unregisters.forEach(u => u())
|
||||
}
|
||||
|
||||
private saveManagedRecord(filename: string, checksum: string) {
|
||||
|
||||
}
|
||||
|
||||
getFileSignal(filename: string): Signal<DataTransformerOut> {
|
||||
if (this.files.has(filename)) {
|
||||
return this.files.get(filename)!
|
||||
|
|
@ -50,12 +67,13 @@ export class FilesManager {
|
|||
const fileSignal = csvFileSignal(filename)
|
||||
const signal = dataTransformer(fileSignal)
|
||||
|
||||
// Check if it's already in the database, if so, we can just load it again.
|
||||
this.db
|
||||
|
||||
// FIXME: add here ability to watch changes.
|
||||
|
||||
this.files.set(filename, signal)
|
||||
this.inputFiles.set(filename, fileSignal)
|
||||
|
||||
// now firing signal for the file
|
||||
this.loadFile(filename).then(fileSignal)
|
||||
|
||||
return signal
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export class TablesManager {
|
|||
|
||||
if (this.tableLinks.has(tableName)) {
|
||||
const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)!
|
||||
if (prevFileName === fileName) {
|
||||
if (prevFileName === file.path) {
|
||||
return
|
||||
}
|
||||
unlink()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { TFile } from "obsidian";
|
||||
import { createSignal, Signal } from "src/utils/signal";
|
||||
|
||||
export type CSVData = string
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { CSVData } from "./csvFile"
|
|||
import { parse } from 'papaparse'
|
||||
import { toTypeStatements } from "../utils"
|
||||
import { sanitise } from "../utils/sanitiseColumn"
|
||||
import { TFile } from "obsidian"
|
||||
|
||||
|
||||
export const parseData = (csvData: string) => {
|
||||
|
|
@ -17,11 +18,9 @@ export const parseData = (csvData: string) => {
|
|||
|
||||
|
||||
export const dataTransformer = (s: Signal<CSVData>) => {
|
||||
const sig = derivedSignal([s], (csvData) => {
|
||||
// FIXME: fix header here.
|
||||
const sig = derivedSignal([s], (data) => {
|
||||
try {
|
||||
const parsed = parseData(csvData)
|
||||
|
||||
const parsed = parseData(data)
|
||||
const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
|
||||
return typeStatements
|
||||
} catch (e) {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { SqlSealDatabase } from "src/database";
|
|||
import { FieldTypes } from "src/utils";
|
||||
|
||||
export const linkTableWithFile = (dataSig: Signal<DataTransformerOut>, tableSignal: Signal<number>, tableName: string, db: SqlSealDatabase) => {
|
||||
return dataSig(({ data, types}) => {
|
||||
return dataSig(({ data, types }) => {
|
||||
|
||||
// Check if the columns are exactly the same. If not, delete and reinstantiate the table.
|
||||
|
||||
|
|
|
|||
219
src/database-worker.ts
Normal file
219
src/database-worker.ts
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
import * as Comlink from "comlink"
|
||||
import initSqlJs from '@jlongster/sql.js';
|
||||
import wasmBinary from '../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm'
|
||||
import { SQLiteFS } from 'absurd-sql';
|
||||
import IndexedDBBackend from '../node_modules/absurd-sql/dist/indexeddb-backend.js';
|
||||
import type { BindParams, Database, Statement } from "sql.js";
|
||||
import { sanitise } from "./utils/sanitiseColumn";
|
||||
import { FieldTypes } from "./utils";
|
||||
|
||||
|
||||
function toObjectArray(stmt: Statement) {
|
||||
const ret = []
|
||||
while (stmt.step()) {
|
||||
ret.push(stmt.getAsObject())
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
function recordToBindParams(record: Record<string, unknown>) {
|
||||
const bindParams = Object.fromEntries(Object.entries(record).map(([key, val]) => ([`@${key}`, val]))) as BindParams
|
||||
return bindParams
|
||||
}
|
||||
|
||||
const formatData = (data: Record<string, any>) => {
|
||||
return Object.keys(data).reduce((ret, key) => {
|
||||
if (typeof data[key] === 'boolean') {
|
||||
return {
|
||||
...ret,
|
||||
[key]: data[key] ? 1 : 0
|
||||
}
|
||||
}
|
||||
if (!data[key]) {
|
||||
return {
|
||||
...ret,
|
||||
[key]: null
|
||||
}
|
||||
}
|
||||
if (typeof data[key] === 'object' || Array.isArray(data[key])) {
|
||||
return {
|
||||
...ret,
|
||||
[key]: JSON.stringify(data[key])
|
||||
}
|
||||
}
|
||||
return {
|
||||
...ret,
|
||||
[key]: data[key]
|
||||
}
|
||||
}, {})
|
||||
}
|
||||
|
||||
// Disabling SharedArrayBuffer - otherwise Absurd-SQL doesn't handle properly it.
|
||||
self.SharedArrayBuffer = undefined as any
|
||||
|
||||
export class WorkerDatabase {
|
||||
|
||||
private db: Database
|
||||
|
||||
constructor() {
|
||||
|
||||
}
|
||||
|
||||
defineCustomFunctions() {
|
||||
this.db.create_function('a', (href: string, name: string) => {
|
||||
const linkObject = {
|
||||
type: 'link',
|
||||
href: href,
|
||||
name: name || href
|
||||
};
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`;
|
||||
});
|
||||
|
||||
this.db.create_function('a', (href: string) => {
|
||||
const linkObject = {
|
||||
type: 'link',
|
||||
href: href,
|
||||
name: href
|
||||
};
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`;
|
||||
});
|
||||
|
||||
this.db.create_function('img', (href: string) => {
|
||||
const imgObject = {
|
||||
type: 'img',
|
||||
href: href
|
||||
}
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})`
|
||||
})
|
||||
|
||||
this.db.create_function('img', (href: string, path: string) => {
|
||||
const imgObject = {
|
||||
type: 'img',
|
||||
path,
|
||||
href
|
||||
}
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})`
|
||||
})
|
||||
|
||||
this.db.create_function('checkbox', (val: string) => {
|
||||
const imgObject = {
|
||||
type: 'checkbox',
|
||||
value: val
|
||||
}
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})`
|
||||
})
|
||||
}
|
||||
|
||||
createTableText(tableName: string, fields: string[]) {
|
||||
const fieldsString = fields.map(f => `${f} TEXT`).join(',')
|
||||
this.db.exec(`CREATE TABLE IF NOT EXISTS ${tableName} (:fieldsString)`, {
|
||||
tableName,
|
||||
fieldsString
|
||||
})
|
||||
}
|
||||
|
||||
async createTable(tableName: string, fields: Record<string, FieldTypes>) {
|
||||
const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
|
||||
const uniqueFields = [...new Map(transformedFiels.map(item =>
|
||||
[item[0], item])).values()]
|
||||
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)
|
||||
// FIXME: probably use schema generator, for now create with hardcoded fields
|
||||
await this.dropTable(tableName)
|
||||
const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} (
|
||||
${sqlFields.join(', ')}
|
||||
);`
|
||||
|
||||
this.db.prepare(createSQL).run()
|
||||
await this.clearTable(tableName)
|
||||
|
||||
// Dropping data.
|
||||
}
|
||||
|
||||
async clearTable(tableName: string) {
|
||||
this.db.prepare(`DELETE FROM ${tableName}`).run()
|
||||
|
||||
}
|
||||
|
||||
async insertData(tableName: string, data: Record<string, unknown>[]) {
|
||||
data.forEach(d => {
|
||||
const columns = Object.keys(d)
|
||||
const insertStatement = this.db.prepare(`INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
|
||||
insertStatement.run(recordToBindParams(formatData(d)))
|
||||
})
|
||||
}
|
||||
|
||||
async dropTable(tableName: string) {
|
||||
this.db.run(`DROP TABLE IF EXISTS ${tableName}`)
|
||||
}
|
||||
|
||||
async select(query: string, params: Record<string, unknown>) {
|
||||
const stmt = this.db.prepare(query, recordToBindParams(params))
|
||||
return {
|
||||
data: toObjectArray(stmt),
|
||||
columns: stmt.getColumnNames()
|
||||
}
|
||||
}
|
||||
|
||||
async updateData(tableName: string, data: Array<Record<string, unknown>>) {
|
||||
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
|
||||
data.forEach((d: Record<string, unknown>) => {
|
||||
const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`)
|
||||
stmt.run(recordToBindParams(d))
|
||||
})
|
||||
}
|
||||
|
||||
async deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
|
||||
data.forEach(d => {
|
||||
const stmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`);
|
||||
stmt.run({
|
||||
[`@${key}`]: d[key]
|
||||
} as BindParams)
|
||||
})
|
||||
}
|
||||
|
||||
async connect() {
|
||||
try {
|
||||
const SQL = await initSqlJs({
|
||||
wasmBinary: wasmBinary
|
||||
});
|
||||
|
||||
let sqlFS = new SQLiteFS(SQL.FS, new IndexedDBBackend(() => {
|
||||
console.error('unable to write to indexedDb')
|
||||
}));
|
||||
SQL.register_for_idb(sqlFS);
|
||||
|
||||
SQL.FS.mkdir('/sql');
|
||||
SQL.FS.mount(sqlFS, {}, '/sql');
|
||||
|
||||
const path = 'sql/sqlseal.sqlite';
|
||||
|
||||
let stream = SQL.FS.open(path, 'a+');
|
||||
await stream.node.contents.readIfFallback();
|
||||
SQL.FS.close(stream);
|
||||
|
||||
const db = new SQL.Database('sql/sqlseal.sqlite', { filename: true });
|
||||
|
||||
let cacheSize = 0;
|
||||
let pageSize = 4096;
|
||||
|
||||
db.exec(`
|
||||
PRAGMA cache_size=-${cacheSize};
|
||||
PRAGMA journal_mode=MEMORY;
|
||||
PRAGMA page_size=${pageSize};
|
||||
VACUUM;
|
||||
`);
|
||||
this.db = db
|
||||
this.defineCustomFunctions()
|
||||
return
|
||||
} catch (e) {
|
||||
console.error('Error while setting up database', e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
// FIXME: implement.
|
||||
}
|
||||
}
|
||||
|
||||
Comlink.expose(WorkerDatabase);
|
||||
200
src/database.ts
200
src/database.ts
|
|
@ -1,14 +1,12 @@
|
|||
import { App } from "obsidian"
|
||||
import path from 'path'
|
||||
import Papa from 'papaparse'
|
||||
import { prefixedIfNotGlobal } from "./sqlReparseTables"
|
||||
import { camelCase } from 'lodash'
|
||||
import { dataToCamelCase, fetchBlobData, FieldTypes, predictJson, predictType, toTypeStatements } from "./utils"
|
||||
import os from 'os'
|
||||
import fs from 'fs'
|
||||
import { FieldTypes, toTypeStatements } from "./utils"
|
||||
import { sanitise } from "./utils/sanitiseColumn"
|
||||
import initSqlJs, { BindParams, Database, Statement } from 'sql.js'
|
||||
import wasmBinary from '../node_modules/sql.js/dist/sql-wasm.wasm'
|
||||
import * as Comlink from 'comlink'
|
||||
import workerCode from 'virtual:worker-code'
|
||||
import { WorkerDatabase } from "./database-worker"
|
||||
|
||||
|
||||
export interface FieldDefinition {
|
||||
name: string;
|
||||
|
|
@ -43,12 +41,9 @@ const formatData = (data: Record<string, any>) => {
|
|||
}
|
||||
|
||||
export class SqlSealDatabase {
|
||||
private savedDatabases: Record<string, any> = {}
|
||||
db: Database
|
||||
private SQL: initSqlJs.SqlJsStatic
|
||||
db: Comlink.Remote<WorkerDatabase>
|
||||
private isConnected = false
|
||||
private connectingPromise: Promise<void>;
|
||||
private connectingPromiseResolve: (value: void | PromiseLike<void>) => void
|
||||
constructor(private readonly app: App, private readonly verbose = false) {
|
||||
|
||||
}
|
||||
|
|
@ -62,76 +57,34 @@ export class SqlSealDatabase {
|
|||
return this.connectingPromise
|
||||
}
|
||||
|
||||
this.connectingPromise = new Promise((resolve) => {
|
||||
this.connectingPromiseResolve = resolve
|
||||
})
|
||||
this.connectingPromise = new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
const blob = new Blob([workerCode], { type: 'text/javascript' });
|
||||
const workerUrl = URL.createObjectURL(blob);
|
||||
|
||||
const worker = new Worker(workerUrl, {
|
||||
name: 'SQLSeal Database'
|
||||
});
|
||||
const DatabaseWrap = Comlink.wrap<typeof WorkerDatabase>(worker)
|
||||
|
||||
try {
|
||||
this.SQL = await initSqlJs({
|
||||
wasmBinary: wasmBinary
|
||||
})
|
||||
|
||||
this.db = new this.SQL.Database()
|
||||
await this.defineCustomFunctions()
|
||||
|
||||
this.isConnected = true
|
||||
this.connectingPromiseResolve()
|
||||
} catch (e) {
|
||||
console.error('Error initializing SQLite database:', e)
|
||||
}
|
||||
}
|
||||
const instance = await new DatabaseWrap()
|
||||
|
||||
private async defineCustomFunctions() {
|
||||
this.db.create_function('a', (href: string, name: string) => {
|
||||
const linkObject = {
|
||||
type: 'link',
|
||||
href: href,
|
||||
name: name || href
|
||||
};
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`;
|
||||
});
|
||||
|
||||
this.db.create_function('a', (href: string) => {
|
||||
const linkObject = {
|
||||
type: 'link',
|
||||
href: href,
|
||||
name: href
|
||||
};
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(linkObject)})`;
|
||||
});
|
||||
|
||||
this.db.create_function('img', (href: string) => {
|
||||
const imgObject = {
|
||||
type: 'img',
|
||||
href: href
|
||||
await instance.connect()
|
||||
this.db = instance
|
||||
this.isConnected = true
|
||||
resolve()
|
||||
} catch (e) {
|
||||
reject(e)
|
||||
}
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})`
|
||||
})
|
||||
|
||||
this.db.create_function('img', (href: string, path: string) => {
|
||||
const imgObject = {
|
||||
type: 'img',
|
||||
path,
|
||||
href
|
||||
}
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})`
|
||||
})
|
||||
|
||||
this.db.create_function('checkbox', (val: string) => {
|
||||
const imgObject = {
|
||||
type: 'checkbox',
|
||||
value: val
|
||||
}
|
||||
return `SQLSEALCUSTOM(${JSON.stringify(imgObject)})`
|
||||
})
|
||||
|
||||
return this.connectingPromise
|
||||
}
|
||||
|
||||
async disconect() {
|
||||
if (!this.isConnected) {
|
||||
return
|
||||
}
|
||||
this.db.close()
|
||||
this.db.disconnect()
|
||||
this.isConnected = false
|
||||
}
|
||||
|
||||
|
|
@ -143,88 +96,47 @@ export class SqlSealDatabase {
|
|||
return schema
|
||||
}
|
||||
|
||||
async addNewColumns(name: string, data: Array<Record<string, unknown>>) {
|
||||
const schema = await this.getSchema(data)
|
||||
const currentSchema = this.toObjectsArray(this.db.prepare(`PRAGMA table_info(${name})`))
|
||||
const currentFields = currentSchema.map((f: any) => f.name)
|
||||
const newFields = Object.keys(schema).filter(f => !currentFields.includes(f))
|
||||
// async addNewColumns(name: string, data: Array<Record<string, unknown>>) {
|
||||
// const schema = await this.getSchema(data)
|
||||
// const currentSchema = this.toObjectsArray(this.db.prepare(`PRAGMA table_info(${name})`))
|
||||
// const currentFields = currentSchema.map((f: any) => f.name)
|
||||
// const newFields = Object.keys(schema).filter(f => !currentFields.includes(f))
|
||||
|
||||
if (newFields.length === 0) {
|
||||
return
|
||||
}
|
||||
// if (newFields.length === 0) {
|
||||
// return
|
||||
// }
|
||||
|
||||
const alter = this.db.prepare(`ALTER TABLE ${name} ADD
|
||||
COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`)
|
||||
alter.run()
|
||||
// const alter = this.db.prepare(`ALTER TABLE ${name} ADD
|
||||
// COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`)
|
||||
// alter.run()
|
||||
// }
|
||||
|
||||
async updateData(name: string, data: Array<Record<string, unknown>>) {
|
||||
return this.db.updateData(name, data)
|
||||
}
|
||||
|
||||
private toObjectsArray(stmt: Statement) {
|
||||
const ret = []
|
||||
while(stmt.step()) {
|
||||
ret.push(stmt.getAsObject())
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
private recordToBindParams(record: Record<string, unknown>) {
|
||||
const bindParams = Object.fromEntries(Object.entries(record).map(([key, val]) => ([`@${key}`, val]))) as BindParams
|
||||
return bindParams
|
||||
}
|
||||
|
||||
updateData(name: string, data: Array<Record<string, unknown>>) {
|
||||
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
|
||||
const update = this.db.prepare(`UPDATE ${name} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`);
|
||||
data.forEach((d: Record<string, unknown>) => {
|
||||
const stmt = this.db.prepare(`UPDATE ${name} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`)
|
||||
stmt.run(this.recordToBindParams(d))
|
||||
})
|
||||
}
|
||||
|
||||
deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
|
||||
data.forEach(d => {
|
||||
const stmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`);
|
||||
stmt.run({
|
||||
[`@${key}`]: d[key]
|
||||
} as BindParams)
|
||||
})
|
||||
async deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
|
||||
return this.db.deleteData(name, data, key)
|
||||
}
|
||||
|
||||
async insertData(name: string, inData: Array<Record<string, unknown>>) {
|
||||
inData.forEach(d => {
|
||||
const columns = Object.keys(d)
|
||||
const insertStatement = this.db.prepare(`INSERT INTO ${name} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
|
||||
insertStatement.run(this.recordToBindParams(formatData(d)))
|
||||
})
|
||||
return this.db.insertData(name, inData)
|
||||
}
|
||||
|
||||
dropTable(name: string) {
|
||||
this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
|
||||
this.savedDatabases[name] = false
|
||||
async dropTable(name: string) {
|
||||
return this.db.dropTable(name)
|
||||
}
|
||||
|
||||
createTableClean(name: string, fields: Array<FieldDefinition>) {
|
||||
const sqlFields = fields.map(({ name, type }) => `${name} ${type}`).join(', ')
|
||||
const createSql = `CREATE TABLE IF NOT EXISTS ${name} (${sqlFields})`
|
||||
this.db.prepare(createSql).run()
|
||||
this.savedDatabases[name] = true
|
||||
async createTableClean(name: string, fields: Array<FieldDefinition>) {
|
||||
const fieldsToRecord = fields.reduce((acc, f) => ({
|
||||
...acc,
|
||||
[f.name]: f.type
|
||||
}), {} as Record<string, FieldTypes>) as Record<string, FieldTypes>
|
||||
await this.createTable(name, fieldsToRecord)
|
||||
}
|
||||
|
||||
async createTable(name: string, fields: Record<string, FieldTypes>) {
|
||||
const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
|
||||
const uniqueFields = [...new Map(transformedFiels.map(item =>
|
||||
[item[0], item])).values()]
|
||||
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)
|
||||
// FIXME: probably use schema generator, for now create with hardcoded fields
|
||||
this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
|
||||
const createSQL = `CREATE TABLE IF NOT EXISTS ${name} (
|
||||
${sqlFields.join(', ')}
|
||||
);`
|
||||
|
||||
this.db.prepare(createSQL).run()
|
||||
this.savedDatabases[name] = true
|
||||
|
||||
// Dropping data.
|
||||
this.db.prepare(`DELETE FROM ${name}`).run()
|
||||
await this.db.createTable(name, fields)
|
||||
}
|
||||
async getSchema(data: Array<Record<string, unknown>>) {
|
||||
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
|
||||
|
|
@ -232,12 +144,8 @@ export class SqlSealDatabase {
|
|||
return types;
|
||||
}
|
||||
|
||||
select(statement: string, frontmatter: Record<string, unknown>) {
|
||||
const stmt = this.db.prepare(statement)
|
||||
stmt.bind(this.recordToBindParams(frontmatter ?? {}))
|
||||
return {
|
||||
data: this.toObjectsArray(stmt) as Record<string, any>[],
|
||||
columns: stmt.getColumnNames()
|
||||
}
|
||||
async select(statement: string, frontmatter: Record<string, unknown>) {
|
||||
const result = await this.db.select(statement, frontmatter)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { SqlSealDatabase } from "src/database";
|
||||
|
||||
|
||||
export interface SyncEntry {
|
||||
filename: string;
|
||||
url: string;
|
||||
|
|
@ -20,40 +21,34 @@ export class SyncModel {
|
|||
createTableIfNotExists() {
|
||||
this.db.createTable(this.tableName, {
|
||||
'filename': 'TEXT',
|
||||
'url': 'TEXT',
|
||||
'name': 'TEXT',
|
||||
'checksum': 'TEXT',
|
||||
'syncedAt': 'TEXT',
|
||||
'metadata': 'JSON'
|
||||
})
|
||||
}
|
||||
|
||||
registerSync(filename: string, url: string, tableName: string) {
|
||||
registerSync(filename: string, checksum: string) {
|
||||
this.db.insertData(this.tableName, [{
|
||||
filename,
|
||||
url,
|
||||
name: tableName,
|
||||
checksum,
|
||||
syncedAt: Date.now(),
|
||||
metadata: { }
|
||||
}])
|
||||
}
|
||||
|
||||
getSync(filename: string, tableName: string) {
|
||||
const { data } = this.db.select(`SELECT *
|
||||
async getSync(filename: string) {
|
||||
const { data } = await this.db.select(`SELECT *
|
||||
FROM sqlseal_sync
|
||||
WHERE filename = @filename
|
||||
AND name = @name`,
|
||||
WHERE filename = @filename`,
|
||||
{
|
||||
name: tableName,
|
||||
filename: filename
|
||||
})
|
||||
|
||||
if (data) {
|
||||
return data[0] as SyncEntry
|
||||
return data[0] as unknown as SyncEntry
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
removeSync(filename: string, tableName: string) {
|
||||
removeSync(filename: string) {
|
||||
// FIXME: do not use exposed db, instead implement "exec" method inside the file.
|
||||
this.db.db.prepare('DELETE FROM sqlseal_sync WHERE filename = :filename AND name = :name').run({
|
||||
filename: filename,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ export class SqlSeal {
|
|||
this.db = new SqlSealDatabase(app, verbose)
|
||||
const logger = new Logger(verbose)
|
||||
|
||||
const fileManager = new FilesManager(this.app.vault, this.app)
|
||||
const fileManager = new FilesManager(this.app.vault, this.app, this.db)
|
||||
this.tablesManager = new TablesManager(fileManager, this.db)
|
||||
this.tablesManager.getTableSignal('files')
|
||||
this.tablesManager.getTableSignal('tags')
|
||||
|
|
|
|||
|
|
@ -13,5 +13,6 @@
|
|||
"0.9.2": "0.15.0",
|
||||
"0.10.0": "0.15.0",
|
||||
"0.10.1": "0.15.0",
|
||||
"0.11.0": "0.15.0"
|
||||
"0.11.0": "0.15.0",
|
||||
"0.12.0": "0.15.0"
|
||||
}
|
||||
Loading…
Reference in a new issue