obsidian-plugin/h-sphere_sql-seal
1
0
Fork 0
mirror of https://github.com/h-sphere/sql-seal.git synced 2026-07-22 10:10:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Migrating SQL Engine to wa-sqlite (#193)

Browse source 
* feat: WIP. Proof of concept of using sqlite-wasm

* chore: few experiments with kysely

* chore: poc of using idb with modern version of sqlite

* feat: reworking project to use wa-sqlite

* feat: making database queries work too

* infra: refactoring database connection and cleaning dependencies

* fix: fixing issue with tests

* chore: adding missing packages

* chore: underlying engine updated to wa-sqlite
This commit is contained in:
Kacper 2026-03-22 18:01:03 +00:00 committed by GitHub
parent bf24096d75
commit bf085fe896
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
60 changed files with 2242 additions and 1664 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/old-wombats-slide.md Normal file
View file

@ -0,0 +1,5 @@
---
"sqlseal": minor
---
underlying engine updated to wa-sqlite

12
.claude/settings.local.json Normal file
View file

@ -0,0 +1,12 @@
{
"permissions": {
"allow": [
"Bash(npm run typecheck:*)",
"Bash(grep:*)",
"Bash(cat:*)",
"WebSearch",
"Bash(node --version:*)",
"Bash(npm test)"
]
}
}

6
docs/contributing/project-architecture.md
View file

@ -2,8 +2,8 @@
SQLSeal is an Obsidian Plugin written in TypeScript. We use the following tech stack:
- [PNPM](https://pnpm.io/)
- [TypeScript](https://www.typescriptlang.org/)
- [SQL.JS](https://github.com/sql-js/sql.js): [SQLite](https://www.sqlite.org/) compiled into WebAssembly
- [AbsurdSQL](https://github.com/jlongster/absurd-sql) to persist SQLite page blocks into IndexedDB and not keep the database in the memory
- [wa-sqlite](https://github.com/rhashimoto/wa-sqlite): [SQLite](https://www.sqlite.org/) compiled into WebAssembly
- IDBBatchAtomicVFS to persist SQLite page blocks into IndexedDB and not keep the database in memory
- [AGGrid](https://www.ag-grid.com/) - grid solution, default table renderer
- Node-SQL-Parser - to parse SQL and modify table names before sending it to SQLite
- Comlink - for Webworker communication abstraction
@ -11,7 +11,7 @@ SQLSeal is an Obsidian Plugin written in TypeScript. We use the following tech s
## Architecture overview
![Architecture Diagram](./diagram.png)
On the high level SQLSeal uses SQLite and communicates with it using WebWorker. The main process calls that Web Worker (using Comlink as a wrapper to abstract away complexity of postMessage and wrap it into class methods returning promises instead). The database is setup in a way that individual blocks are being persisted into the IndexedDB. Thanks to that even if the database grows, the memory footprint should stay relatively low.
On the high level SQLSeal uses SQLite (via wa-sqlite) and communicates with it using WebWorker. The main process calls that Web Worker (using Comlink as a wrapper to abstract away complexity of postMessage and wrap it into class methods returning promises instead). The database is setup in a way that individual blocks are being persisted into IndexedDB using IDBBatchAtomicVFS. Thanks to that even if the database grows, the memory footprint should stay relatively low.
## Extensible Architecture

108
esbuild.config.mjs
View file

@ -16,6 +16,7 @@ if you want to view the source, please visit the github repository of this plugi
const wasmPlugin = {
name: 'wasm',
setup(build) {
// Handle .wasm files
build.onResolve({ filter: /\.wasm$/ }, args => {
if (args.resolveDir === '') return;
return {
@ -37,6 +38,52 @@ const wasmPlugin = {
loader: 'js',
};
});
// Handle SQLite bundler-friendly module
build.onResolve({ filter: /sqlite3-bundler-friendly\.mjs$/ }, args => {
if (args.path.startsWith('@sqlite.org/sqlite-wasm')) {
// Resolve to the actual file path in node_modules
return {
path: join(process.cwd(), 'node_modules', args.path),
namespace: 'sqlite-bundler',
};
}
return {
path: join(args.resolveDir, args.path),
namespace: 'sqlite-bundler',
};
});
build.onLoad({ filter: /sqlite3-bundler-friendly\.mjs$/, namespace: 'sqlite-bundler' }, async (args) => {
const moduleContents = readFileSync(args.path, 'utf8');
const wasmPath = args.path.replace('sqlite3-bundler-friendly.mjs', 'sqlite3.wasm');
const wasmContents = readFileSync(wasmPath);
const wasmBase64 = wasmContents.toString('base64');
// Replace the findWasmBinary function to return embedded WASM
const modifiedContents = moduleContents
// Replace the entire findWasmBinary function
.replace(
/function\s+findWasmBinary\s*\(\s*\)\s*\{[\s\S]*?return\s+new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)\.href\s*;?\s*\}/g,
`function findWasmBinary() {
return "data:application/wasm;base64,${wasmBase64}";
}`
)
// Also replace any direct new URL(...) patterns as fallback
.replace(
/new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)\.href/g,
`"data:application/wasm;base64,${wasmBase64}"`
)
.replace(
/new\s+URL\s*\(\s*['"`]sqlite3\.wasm['"`]\s*,\s*import\.meta\.url\s*\)/g,
`"data:application/wasm;base64,${wasmBase64}"`
);
return {
contents: modifiedContents,
loader: 'js',
};
});
}
};
@ -44,21 +91,46 @@ const wasmPlugin = {
const workerPlugin = {
name: 'worker',
setup(build) {
build.onResolve({ filter: /^virtual:worker-code$/ }, args => ({
// Handle sqlocal worker code
build.onResolve({ filter: /^virtual:sqlocal-worker-code$/ }, args => ({
path: args.path,
namespace: 'worker-code',
namespace: 'sqlocal-worker-code',
}));
build.onLoad({ filter: /.*/, namespace: 'worker-code' }, async () => {
// Build worker code
build.onLoad({ filter: /.*/, namespace: 'sqlocal-worker-code' }, async () => {
// Create a plugin for the worker that resolves virtual imports
const workerVirtualPlugin = {
name: 'worker-virtual',
setup(build) {
// Handle virtual WASM URL for wa-sqlite
build.onResolve({ filter: /^virtual:wa-sqlite-wasm-url$/ }, args => ({
path: args.path,
namespace: 'wa-sqlite-wasm-url',
}));
build.onLoad({ filter: /.*/, namespace: 'wa-sqlite-wasm-url' }, async () => {
const wasmPath = join(process.cwd(), 'node_modules/wa-sqlite/dist/wa-sqlite-async.wasm');
const wasmContents = readFileSync(wasmPath);
const wasmBase64 = wasmContents.toString('base64');
const wasmDataUrl = `data:application/wasm;base64,${wasmBase64}`;
return {
contents: `export default ${JSON.stringify(wasmDataUrl)};`,
loader: 'js',
};
});
}
};
// Build sqlocal worker code
const result = await esbuild.build({
entryPoints: ['src/modules/database/worker/database.ts'],
entryPoints: ['src/modules/database/sqlocal/sqlocalWorkerDatabase.ts'],
bundle: true,
write: false,
format: 'iife',
target: 'es2020',
external: ['fs', 'path'],
plugins: [wasmPlugin, polyfillNode({
external: ['fs', 'path', 'obsidian'],
plugins: [wasmPlugin, workerVirtualPlugin, polyfillNode({
})],
minify: process.argv[2] === 'production',
define: {
@ -69,12 +141,30 @@ const workerPlugin = {
return {
contents: `
const workerCode = ${JSON.stringify(result.outputFiles[0].text)};
export default workerCode;
const sqlocalWorkerCode = ${JSON.stringify(result.outputFiles[0].text)};
export default sqlocalWorkerCode;
`,
loader: 'js',
};
});
// Handle virtual WASM URL for wa-sqlite
build.onResolve({ filter: /^virtual:wa-sqlite-wasm-url$/ }, args => ({
path: args.path,
namespace: 'wa-sqlite-wasm-url',
}));
build.onLoad({ filter: /.*/, namespace: 'wa-sqlite-wasm-url' }, async () => {
const wasmPath = join(process.cwd(), 'node_modules/wa-sqlite/dist/wa-sqlite-async.wasm');
const wasmContents = readFileSync(wasmPath);
const wasmBase64 = wasmContents.toString('base64');
const wasmDataUrl = `data:application/wasm;base64,${wasmBase64}`;
return {
contents: `export default ${JSON.stringify(wasmDataUrl)};`,
loader: 'js',
};
});
},
};

8
jest.config.js
View file

@ -1,8 +0,0 @@
/** @type {import('ts-jest').JestConfigWithTsJest} **/
module.exports = {
testEnvironment: "node",
transform: {
"^.+.tsx?$": ["ts-jest",{}],
"^.+.ts?$": ["ts-jest",{}],
},
};

23
jest.config.mjs Normal file
View file

@ -0,0 +1,23 @@
/** @type {import('ts-jest').JestConfigWithTsJest} **/
export default {
testEnvironment: "node",
extensionsToTreatAsEsm: ['.ts'],
setupFilesAfterEnv: ['<rootDir>/jest.setup.mjs'],
moduleNameMapper: {
'^virtual:wa-sqlite-wasm-url$': '<rootDir>/src/modules/explorer/database/__tests__/__mocks__/wa-sqlite-wasm-url.ts',
'^(\\.{1,2}/.*)\\.js$': '$1',
},
transform: {
'^.+\\.tsx?$': [
'ts-jest',
{
useESM: true,
}
],
},
testPathIgnorePatterns: [
'/node_modules/',
'/__mocks__/'
],
transformIgnorePatterns: [],
};

39
jest.setup.mjs Normal file
View file

@ -0,0 +1,39 @@
// Jest setup file for ESM mode
// Polyfill fetch to handle file:// URLs for wa-sqlite WASM loading
import { readFileSync } from 'fs';
import { fileURLToPath } from 'url';
const originalFetch = global.fetch;
global.fetch = async function (url, options) {
// Handle file:// URLs by reading from filesystem
if (typeof url === 'string' && url.startsWith('file://')) {
try {
const filePath = fileURLToPath(url);
const buffer = readFileSync(filePath);
// Return a proper Response object
return new Response(buffer, {
status: 200,
statusText: 'OK',
headers: {
'Content-Type': 'application/wasm',
'Content-Length': buffer.length.toString(),
},
});
} catch (error) {
return new Response(null, {
status: 404,
statusText: 'Not Found',
});
}
}
// Fall back to original fetch for http:// and https://
if (originalFetch) {
return originalFetch(url, options);
}
throw new Error('fetch is not available');
};

14
package.json
View file

@ -8,12 +8,11 @@
"typecheck": "tsc -noEmit -skipLibCheck",
"build": "node --no-warnings esbuild.config.mjs production",
"version": "node version-bump.mjs && git add manifest.json versions.json",
"rebuild": "electron-rebuild -f -w better-sqlite3",
"release": "pnpm run build && changeset publish",
"docs:dev": "vitepress dev docs",
"docs:build": "vitepress build docs",
"docs:preview": "vitepress preview docs",
"test": "jest",
"test": "NODE_OPTIONS=--experimental-vm-modules jest",
"ci:publish": "./scripts/tag-and-publish.sh",
"ci:version": "changeset version && pnpm run version"
},
@ -27,17 +26,17 @@
"license": "MIT",
"devDependencies": {
"@changesets/cli": "^2.29.5",
"@jest/globals": "^30.3.0",
"@types/esprima": "^4.0.6",
"@types/estraverse": "^5.1.7",
"@types/jest": "^30.0.0",
"@types/lodash": "^4.17.20",
"@types/node": "^24.2.1",
"@types/papaparse": "^5.3.16",
"@types/sql.js": "^1.4.9",
"@types/unidecode": "^1.1.0",
"@typescript-eslint/eslint-plugin": "8.39.1",
"@typescript-eslint/parser": "8.39.1",
"builtin-modules": "5.0.0",
"electron-rebuild": "^3.2.9",
"esbuild": "0.25.9",
"esbuild-plugin-replace": "^1.4.0",
"esbuild-sass-plugin": "^3.3.1",
@ -55,13 +54,11 @@
"@codemirror/language": "^6.11.2",
"@codemirror/state": "^6.5.2",
"@codemirror/view": "^6.38.1",
"@hypersphere/dity": "^0.1.3",
"@hypersphere/dity": "^0.1.4",
"@hypersphere/dity-graph": "^0.0.11",
"@hypersphere/omnibus": "^0.1.6",
"@jlongster/sql.js": "^1.6.7",
"@types/jsonpath": "^0.2.4",
"@vanakat/plugin-api": "^0.2.1",
"absurd-sql": "^0.0.54",
"ag-grid-community": "^34.1.1",
"comlink": "^4.4.2",
"esbuild-plugin-polyfill-node": "^0.3.0",
@ -78,6 +75,7 @@
"sql-parser-cst": "^0.33.1",
"unidecode": "^1.1.0",
"util": "^0.12.5",
"uuid": "^11.1.0"
"uuid": "^11.1.0",
"wa-sqlite": "^1.0.0"
}
}

1310
pnpm-lock.yaml
View file

File diff suppressed because it is too large Load diff

6
src/main.ts
View file

@ -14,7 +14,7 @@ ModuleRegistry.registerModules([AllCommunityModule]);
provideGlobalGridOptions({ theme: "legacy" });
export default class SqlSealPlugin extends Plugin {
container: ReturnType<(typeof mainModule)["build"]>;
container?: ReturnType<(typeof mainModule)["build"]>;
async onload() {
// CONTAINER
@ -24,7 +24,7 @@ export default class SqlSealPlugin extends Plugin {
.resolve('obsidian.vault', d => d.value(this.app.vault))
.build()
const init = await this.container.get("init")
init()
const init = await this.container.get("init");
init();
}
}

4
src/modules/api/init.ts
View file

@ -4,7 +4,7 @@ import {
SQLSealRegisterApi,
} from "./pluginApi/sqlSealApi";
import { RendererRegistry } from "../editor/renderer/rendererRegistry";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser";
const SQLSEAL_API_KEY = "___sqlSeal";
@ -14,7 +14,7 @@ export const apiInit = (
plugin: Plugin,
cellParser: ModernCellParser,
rendererRegistry: RendererRegistry,
db: SqlSealDatabase,
db: SqlocalDatabaseProxy,
) => {
return () => {
const api = new SQLSealRegisterApi(

4
src/modules/api/module.ts
View file

@ -1,7 +1,7 @@
import { Registrator } from "@hypersphere/dity";
import { apiInit } from "./init";
import { Plugin } from "obsidian";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
import { RendererRegistry } from "../editor/renderer/rendererRegistry";
import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser";
@ -9,7 +9,7 @@ import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser
export const apiModule = new Registrator()
.import<'plugin', Plugin>()
.import<'cellParser', Promise<ModernCellParser>>()
.import<'db', Promise<SqlSealDatabase>>()
.import<'db', Promise<SqlocalDatabaseProxy>>()
.import<'rendererRegistry', RendererRegistry>()
.register('init', db => db.fn(apiInit).inject('plugin', 'cellParser', 'rendererRegistry', 'db'))
.export('init')

6
src/modules/api/pluginApi/sqlSealApi.ts
View file

@ -3,7 +3,7 @@ import { version } from '../../../../package.json'
import { RendererConfig, RendererRegistry } from "../../editor/renderer/rendererRegistry";
import { FilepathHasher } from "../../../utils/hasher";
import { DatabaseTable } from "./table";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { ModernCellParser } from "../../syntaxHighlight/cellParser/ModernCellParser";
import { CellFunction } from "../../syntaxHighlight/cellParser/CellFunction";
@ -22,7 +22,7 @@ export class SQLSealRegisterApi {
sqlSealPlugin: Plugin,
private readonly cellParser: ModernCellParser,
private readonly rendererRegistry: RendererRegistry,
private readonly db: SqlSealDatabase
private readonly db: SqlocalDatabaseProxy
) {
sqlSealPlugin.register(() => {
this.registeredApis.forEach(p => {
@ -75,7 +75,7 @@ export class SQLSealApi {
private readonly plugin: Plugin,
private readonly cellParser: ModernCellParser,
private readonly rendererRegistry: RendererRegistry,
private readonly db: SqlSealDatabase
private readonly db: SqlocalDatabaseProxy
) {
plugin.register(() => {
this.unregister()

4
src/modules/api/pluginApi/table.ts
View file

@ -1,9 +1,9 @@
import { SqlSealDatabase } from "../../database/database"
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy"
type Item<Columns extends string[]> = Record<Columns[number], string | number | undefined>
export class DatabaseTable<const Columns extends string[]> {
constructor(private readonly db: SqlSealDatabase ,public readonly tableName: string, private columns: Columns) {
constructor(private readonly db: SqlocalDatabaseProxy ,public readonly tableName: string, private columns: Columns) {
}

130
src/modules/database/database.ts
View file

@ -1,130 +0,0 @@
import { App } from "obsidian"
import * as Comlink from 'comlink'
// @ts-ignore
import workerCode from 'virtual:worker-code'
import { WorkerDatabase } from "./worker/database";
import { ColumnDefinition } from "../../utils/types";
import { sanitise } from "../../utils/sanitiseColumn";
export class SqlSealDatabase {
db?: Comlink.Remote<WorkerDatabase>
private isConnected = false
private connectingPromise?: Promise<void>;
constructor(private readonly app: App) {
}
registerCustomFunction(name: string, argsCount = 1) {
return this.db?.registerCustomFunction(name, argsCount)
}
async connect() {
if (this.isConnected) {
return Promise.resolve()
}
if (this.connectingPromise) {
return this.connectingPromise
}
this.connectingPromise = new Promise(async (resolve, reject) => {
try {
const blob = new Blob([workerCode], { type: 'text/javascript' });
const workerUrl = URL.createObjectURL(blob);
const worker = new Worker(workerUrl, {
name: 'SQLSeal Database'
});
const DatabaseWrap = Comlink.wrap<typeof WorkerDatabase>(worker)
const filename = sanitise(this.app.vault.getName()) + '___' + (this.app as any).appId
const instance = await new DatabaseWrap(filename)
await instance.connect()
this.db = instance
this.isConnected = true
resolve()
} catch (e) {
reject(e)
}
})
return this.connectingPromise
}
async disconect() {
if (!this.isConnected) {
return
}
this.db?.disconnect()
this.isConnected = false
}
async recreateDatabase() {
await this.db?.recreateDatabase()
}
async updateData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
return this.db?.updateData(name, data, key)
}
async deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
return this.db?.deleteData(name, data, key)
}
async insertData(name: string, inData: Array<Record<string, unknown>>) {
return this.db?.insertData(name, inData)
}
async dropTable(name: string) {
return this.db?.dropTable(name)
}
async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) {
await this.db?.createTableNoTypes(name, columns, noDrop)
}
async createTable(name: string, columns: ColumnDefinition[], noDrop?: boolean) {
this.db?.createTable(name, columns, noDrop)
}
async createIndex(indexName: string, tableName: string, columns: string[]) {
await this.db?.createIndex(indexName, tableName, columns)
}
async getColumns(tableName: string) {
return this.db?.getColumns(tableName)
}
async count(tableName: string) {
const data = await this.db?.select(`SELECT COUNT(*) as count FROM ${tableName}`, {})
return data?.data[0]['count']
}
async addColumns(tableName: string, newColumns: string[]) {
return this.db?.addColumns(tableName, newColumns)
}
async select(statement: string, frontmatter: Record<string, unknown>) {
const result = await this.db?.select(statement, frontmatter)
return result
}
async explain(statement: string, frontmatter: Record<string, unknown>) {
const explainResults = await this.db?.explainQuery(statement, frontmatter)
let strResult = '';
const map = new Map<number, number>()
const INDENT_INCREASE = 4
map.set(0, -INDENT_INCREASE)
for (const result of explainResults || []) {
const parent = parseInt((result.parent as unknown as string) ?? '0', 10)
const indent = map.get(parent)! + INDENT_INCREASE
for (let i=0;i<indent;i++) {
strResult += ' '
}
strResult += result.detail + "\n"
map.set(result.id as number, indent)
}
return strResult
}
}

9
src/modules/database/factory.ts
View file

@ -1,8 +1,11 @@
import { App } from "obsidian";
import { SqlSealDatabase } from "./database";
import { DatabaseProvider } from "./sqlocal";
export const databaseFactory = async (app: App) => {
const db = new SqlSealDatabase(app)
export const databaseFactory = async (app: App, provider: DatabaseProvider) => {
console.log('#### DATABASE CREATION')
const db = await provider.get(null)
await db.connect()
return db
}

11
src/modules/database/module.ts
View file

@ -1,8 +1,13 @@
import { Registrator } from "@hypersphere/dity";
import { App } from "obsidian";
import { DatabaseProvider } from "./sqlocal/databaseProvider";
import { Registrator } from "@hypersphere/dity";
import { databaseFactory } from "./factory";
export type DatabaseModule = typeof db
export const db = new Registrator()
.import<'app', App>()
.register('db', d => d.fn(databaseFactory).inject('app'))
.export('db')
.register('provider', d => d.cls(DatabaseProvider).inject('app'))
.register('db', d => d.fn(databaseFactory).inject('app', 'provider'))
.export('db', 'provider')

52
src/modules/database/sqlocal/databaseProvider.ts Normal file
View file

@ -0,0 +1,52 @@
import { App } from "obsidian";
import { sanitise } from "../../../utils/sanitiseColumn";
import { SqlocalDatabaseProxy } from "./sqlocalDatabaseProxy";
export class DatabaseProvider {
private databases: Map<string, SqlocalDatabaseProxy> = new Map();
constructor(private app: App) { }
get prefix() {
const filename = `sqlseal_1__` +
sanitise(this.app.vault.getName()) + "___" + (this.app as any).appId;
return filename;
}
async get(filename: string | null = null): Promise<SqlocalDatabaseProxy> {
const key = filename ?? "GLOBAL";
// Return cached instance if it exists
const existing = this.databases.get(key);
if (existing) {
return existing;
}
// Create new instance with error handling
const dbName = filename ? `${this.prefix}_${filename}` : `${this.prefix}_db`;
try {
// Create the proxy which will handle worker creation and communication
const sqlocalDb = new SqlocalDatabaseProxy(this.app, dbName);
// Connect to initialize the worker
await sqlocalDb.connect();
// Cache the instance
this.databases.set(key, sqlocalDb);
return sqlocalDb;
} catch (error) {
console.error('DatabaseProvider: Failed to initialize database:', dbName, error);
// If there's a cached instance that failed, remove it
this.databases.delete(key);
throw error;
}
}
async close() {
const promises = Array.from(this.databases.values()).map(db => db.disconnect());
await Promise.all(promises);
this.databases.clear();
}
}

2
src/modules/database/sqlocal/index.ts Normal file
View file

@ -0,0 +1,2 @@
export { SqlocalDatabaseProxy } from './sqlocalDatabaseProxy';
export { DatabaseProvider } from './databaseProvider';

40
src/modules/database/sqlocal/schema.ts Normal file
View file

@ -0,0 +1,40 @@
export type DatabaseSchema = {
files: Files
links: Links
tags: Tags
tasks: Tasks
}
export type Files = {
id: string
name: string
path: string
created_at: Date
modified_at: Date
file_size: number
}
export type Links = {
path: string
target: string
position: string
display_text: string
target_exists: boolean
}
export type Tags = {
tag: string
fileId: string
path: string
}
export type Tasks = {
checkbox: string
task: string
completed: string
filePath: string
path: string
position: string
heading: string
heading_level: string
}

138
src/modules/database/sqlocal/sqlocalDatabaseProxy.ts Normal file
View file

@ -0,0 +1,138 @@
import { App } from "obsidian";
import * as Comlink from 'comlink';
import { SqlocalWorkerDatabase } from "./sqlocalWorkerDatabase";
import { ColumnDefinition } from "../../../utils/types";
import { sanitise } from "../../../utils/sanitiseColumn";
/**
* Main-thread proxy for SqlocalWorkerDatabase.
* All database operations are executed in a Web Worker to avoid blocking the main thread.
*
* This class has the same API as SqlocalDatabase but communicates with the worker
* via Comlink (using postMessage under the hood).
*/
export class SqlocalDatabaseProxy {
private db?: Comlink.Remote<SqlocalWorkerDatabase>;
private isConnected = false;
private connectingPromise?: Promise<void>;
private worker?: Worker;
constructor(private readonly app: App, private readonly dbName: string) {
}
async connect() {
if (this.isConnected) {
return Promise.resolve();
}
if (this.connectingPromise) {
return this.connectingPromise;
}
this.connectingPromise = new Promise(async (resolve, reject) => {
try {
// Import the worker code built by esbuild
// @ts-ignore
const workerCodeModule = await import('virtual:sqlocal-worker-code');
const workerCode = workerCodeModule.default;
// Create worker from blob
const blob = new Blob([workerCode], { type: 'text/javascript' });
const workerUrl = URL.createObjectURL(blob);
this.worker = new Worker(workerUrl, {
name: 'SQLSeal Sqlocal Database'
});
// Wrap worker with Comlink
const DatabaseWrap = Comlink.wrap<typeof SqlocalWorkerDatabase>(this.worker);
const instance = await new DatabaseWrap(this.dbName);
await instance.connect();
this.db = instance;
this.isConnected = true;
resolve();
} catch (e) {
console.error('SqlocalDatabaseProxy: Failed to initialize worker database:', e);
reject(e);
}
});
return this.connectingPromise;
}
async disconnect() {
if (!this.isConnected) {
return;
}
await this.db?.disconnect();
if (this.worker) {
this.worker.terminate();
this.worker = undefined;
}
this.db = undefined;
this.isConnected = false;
}
registerCustomFunction(name: string, argsCount = 1) {
return this.db?.registerCustomFunction(name, argsCount);
}
async recreateDatabase() {
return this.db?.recreateDatabase();
}
async updateData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
return this.db?.updateData(name, data, key);
}
async deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
return this.db?.deleteData(name, data, key);
}
async insertData(name: string, inData: Array<Record<string, unknown>>) {
return this.db?.insertData(name, inData);
}
async dropTable(name: string) {
return this.db?.dropTable(name);
}
async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) {
return this.db?.createTableNoTypes(name, columns, noDrop);
}
async createTable(name: string, columns: ColumnDefinition[], noDrop?: boolean) {
return this.db?.createTable(name, columns, noDrop);
}
async createIndex(indexName: string, tableName: string, columns: string[]) {
return this.db?.createIndex(indexName, tableName, columns);
}
async getColumns(name: string) {
return this.db?.getColumns(name);
}
async count(tableName: string) {
return this.db?.count(tableName);
}
async addColumns(tableName: string, newColumns: string[]) {
return this.db?.addColumns(tableName, newColumns);
}
async select(statement: string, frontmatter: Record<string, unknown>) {
return this.db?.select(statement, frontmatter);
}
async explain(statement: string, frontmatter: Record<string, unknown>) {
return this.db?.explain(statement, frontmatter) ?? "";
}
async hasTable(tableName: string) {
return this.db?.hasTable(tableName);
}
}

712
src/modules/database/sqlocal/sqlocalWorkerDatabase.ts Normal file
View file

@ -0,0 +1,712 @@
import * as Comlink from "comlink";
import SQLiteAsyncESMFactory from 'wa-sqlite/dist/wa-sqlite-async.mjs';
import * as SQLite from 'wa-sqlite';
import { IDBBatchAtomicVFS } from 'wa-sqlite/src/examples/IDBBatchAtomicVFS.js';
import { ColumnDefinition } from "../../../utils/types";
import { sanitise } from "../../../utils/sanitiseColumn";
// Get the WASM URL from the virtual module
// @ts-ignore
import wasmUrl from 'virtual:wa-sqlite-wasm-url';
/**
* Worker-side database implementation that runs wa-sqlite operations
* in a Web Worker to avoid blocking the main thread.
*
* This class is exposed via Comlink and all methods are called from the main thread.
*/
export class SqlocalWorkerDatabase {
private connection: number | null = null;
private sqlite3: any = null;
private isConnected = false;
private vfsRegistered = false;
private isRecreating = false;
constructor(private readonly dbName: string) {
}
/**
* Format data for SQLite storage (matching old sql.js implementation)
* - Converts booleans to 0/1
* - Converts falsy values to null
* - JSON-stringifies objects and arrays
*/
private formatData(data: Record<string, any>): Record<string, any> {
return Object.keys(data).reduce((ret, key) => {
const value = data[key];
// Convert booleans to 0/1 for SQLite
if (typeof value === 'boolean') {
return { ...ret, [key]: value ? 1 : 0 };
}
// Convert falsy values to null
if (!value) {
return { ...ret, [key]: null };
}
// JSON-stringify objects and arrays
if (typeof value === 'object' || Array.isArray(value)) {
return { ...ret, [key]: JSON.stringify(value) };
}
// Pass through other values as-is
return { ...ret, [key]: value };
}, {});
}
private async initializeSQLite() {
if (this.sqlite3) {
return this.sqlite3;
}
try {
// Initialize the module with bundled WASM
const asyncModule = await SQLiteAsyncESMFactory({
locateFile: (file: string) => {
if (file.endsWith('.wasm')) {
return wasmUrl;
}
return file;
}
});
// Use Factory to get the actual sqlite3 API
this.sqlite3 = SQLite.Factory(asyncModule);
return this.sqlite3;
} catch (error) {
console.error('SqlocalWorkerDatabase: Failed to initialize wa-sqlite:', error);
throw error;
}
}
private registerVFS(dbName: string) {
if (this.vfsRegistered) {
return;
}
try {
// Register VFS with relaxed durability for performance
// @ts-ignore - Constructor does accept these parameters despite TypeScript definition
const vfs = new IDBBatchAtomicVFS(dbName, {
durability: "relaxed"
});
this.sqlite3.vfs_register(vfs);
this.vfsRegistered = true;
} catch (error) {
console.error('SqlocalWorkerDatabase: Failed to register VFS:', error);
throw error;
}
}
async connect() {
if (this.isConnected) {
return Promise.resolve();
}
try {
// Initialize SQLite
await this.initializeSQLite();
// Register VFS
this.registerVFS(this.dbName);
// Open database connection with concurrent read support
this.connection = await this.sqlite3.open_v2(
this.dbName,
SQLite.SQLITE_OPEN_CREATE | SQLite.SQLITE_OPEN_READWRITE
);
// Configure for optimal speed (data can be recreated in this project)
// Using MEMORY journal mode for maximum performance
await this.sqlite3.exec(this.connection, `
PRAGMA journal_mode=MEMORY;
PRAGMA synchronous=OFF;
PRAGMA cache_size=10000;
PRAGMA temp_store=MEMORY;
PRAGMA page_size=4096;
`);
this.isConnected = true;
} catch (error) {
console.error('SqlocalWorkerDatabase: Failed to connect:', error);
throw error;
}
}
async disconnect() {
if (!this.isConnected || !this.connection) {
return;
}
if (this.connection && this.sqlite3) {
await this.sqlite3.close(this.connection);
this.connection = null;
this.isConnected = false;
}
}
registerCustomFunction(name: string, argsCount = 1) {
if (!this.connection) {
console.warn('SqlocalWorkerDatabase: Database not connected, cannot register custom function');
return Promise.resolve();
}
// Register different overloads based on argument count using wa-sqlite API
// The callback signature is (context: number, values: Uint32Array)
// where values is an array of pointers to sqlite3_value
try {
if (argsCount >= 1) {
this.sqlite3.create_function(
this.connection,
name,
1,
this.sqlite3.SQLITE_UTF8,
null,
(context: number, values: Uint32Array) => {
const arg0 = this.sqlite3.value_text(values[0]);
const data = { type: name, values: [arg0] };
const result = `SQLSEALCUSTOM(${JSON.stringify(data)})`;
this.sqlite3.result_text(context, result);
}
);
}
if (argsCount >= 2) {
this.sqlite3.create_function(
this.connection,
name,
2,
this.sqlite3.SQLITE_UTF8,
null,
(context: number, values: Uint32Array) => {
const arg0 = this.sqlite3.value_text(values[0]);
const arg1 = this.sqlite3.value_text(values[1]);
const data = { type: name, values: [arg0, arg1] };
const result = `SQLSEALCUSTOM(${JSON.stringify(data)})`;
this.sqlite3.result_text(context, result);
}
);
}
if (argsCount >= 3) {
this.sqlite3.create_function(
this.connection,
name,
3,
this.sqlite3.SQLITE_UTF8,
null,
(context: number, values: Uint32Array) => {
const arg0 = this.sqlite3.value_text(values[0]);
const arg1 = this.sqlite3.value_text(values[1]);
const arg2 = this.sqlite3.value_text(values[2]);
const data = { type: name, values: [arg0, arg1, arg2] };
const result = `SQLSEALCUSTOM(${JSON.stringify(data)})`;
this.sqlite3.result_text(context, result);
}
);
}
if (argsCount >= 4) {
throw new Error('Too many arguments, only up to 3 arguments are supported at the moment.');
}
} catch (error) {
console.error(`SqlocalWorkerDatabase: Error registering function '${name}':`, error);
}
return Promise.resolve();
}
async recreateDatabase() {
if (!this.connection) throw new Error('Database not connected');
if (this.isRecreating) {
return;
}
try {
this.isRecreating = true;
// Get all table names using wa-sqlite API
const tables: any[] = [];
const sql = `
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
while (await this.sqlite3.step(prepared.stmt) === SQLite.SQLITE_ROW) {
tables.push({ name: await this.sqlite3.column(prepared.stmt, 0) });
}
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
// Drop all tables
for (const table of tables) {
await this.dropTableInternal(table.name);
}
// Run VACUUM to reclaim space and optimize database (matching old sql.js implementation)
await this.sqlite3.exec(this.connection, 'VACUUM');
// Run integrity check to verify database health
await this.sqlite3.exec(this.connection, 'PRAGMA integrity_check');
} catch (error) {
console.error('SqlocalWorkerDatabase: Error during recreateDatabase:', error);
throw error;
} finally {
this.isRecreating = false;
}
}
private async dropTableInternal(name: string) {
const sql = `DROP TABLE IF EXISTS "${name}"`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
await this.sqlite3.step(prepared.stmt);
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
}
async updateData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
if (!this.connection) throw new Error('Database not connected');
if (data.length === 0) {
return;
}
// Format first row to determine columns structure
const firstFormatted = this.formatData(data[0]);
const columns = Object.keys(firstFormatted).filter(k => k !== key);
const setClause = columns.map(col => `"${col}" = ?`).join(', ');
const sql = `UPDATE "${name}" SET ${setClause} WHERE "${key}" = ?`;
// PERFORMANCE OPTIMIZATION: Prepare statement once, reuse for all rows
const str = this.sqlite3.str_new(this.connection, sql);
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (!prepared) {
this.sqlite3.str_finish(str);
throw new Error('Failed to prepare update statement');
}
try {
for (const row of data) {
// Format data before updating
const formattedRow = this.formatData(row);
const keyValue = formattedRow[key];
const values = columns.map(col => formattedRow[col]);
// Reset statement for reuse instead of preparing again
await this.sqlite3.reset(prepared.stmt);
await this.sqlite3.bind_collection(prepared.stmt, [...values, keyValue]);
await this.sqlite3.step(prepared.stmt);
}
} finally {
// Finalize once at the end
await this.sqlite3.finalize(prepared.stmt);
this.sqlite3.str_finish(str);
}
}
async deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
if (!this.connection) throw new Error('Database not connected');
if (data.length === 0) {
return;
}
const sql = `DELETE FROM "${name}" WHERE "${key}" = ?`;
// PERFORMANCE OPTIMIZATION: Prepare statement once, reuse for all rows
const str = this.sqlite3.str_new(this.connection, sql);
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (!prepared) {
this.sqlite3.str_finish(str);
throw new Error('Failed to prepare delete statement');
}
try {
for (const row of data) {
// Format data for consistency (though only key is used)
const formattedRow = this.formatData(row);
const keyValue = formattedRow[key];
// Reset statement for reuse instead of preparing again
await this.sqlite3.reset(prepared.stmt);
await this.sqlite3.bind_collection(prepared.stmt, [keyValue]);
await this.sqlite3.step(prepared.stmt);
}
} finally {
// Finalize once at the end
await this.sqlite3.finalize(prepared.stmt);
this.sqlite3.str_finish(str);
}
}
async insertData(name: string, inData: Array<Record<string, unknown>>) {
if (!this.connection) throw new Error('Database not connected');
if (inData.length === 0) {
return;
}
// Filter out __parsed_extra column (internal metadata) - matches old sql.js implementation
const columns = Object.keys(inData[0]).filter(c => c !== '__parsed_extra');
const placeholders = columns.map(() => '?').join(', ');
const columnNames = columns.map(col => `"${col}"`).join(', ');
const sql = `INSERT INTO "${name}" (${columnNames}) VALUES (${placeholders})`;
// PERFORMANCE OPTIMIZATION: Prepare statement once, reuse for all rows
const str = this.sqlite3.str_new(this.connection, sql);
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (!prepared) {
this.sqlite3.str_finish(str);
throw new Error('Failed to prepare insert statement');
}
try {
for (const row of inData) {
// Format data (booleans → 0/1, objects → JSON) before inserting
const formattedRow = this.formatData(row);
const values = columns.map(col => formattedRow[col]);
// Reset statement for reuse instead of preparing again
await this.sqlite3.reset(prepared.stmt);
await this.sqlite3.bind_collection(prepared.stmt, values);
await this.sqlite3.step(prepared.stmt);
}
} finally {
// Finalize once at the end
await this.sqlite3.finalize(prepared.stmt);
this.sqlite3.str_finish(str);
}
}
async dropTable(name: string) {
if (!this.connection) throw new Error('Database not connected');
try {
await this.dropTableInternal(name);
} catch (error) {
console.error(`SqlocalWorkerDatabase: Error dropping table '${name}':`, error);
throw error;
}
}
async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) {
if (!this.connection) throw new Error('Database not connected');
if (!noDrop) {
await this.dropTableInternal(name);
}
const columnDefs = columns.map(col => `"${col}" TEXT`).join(', ');
const sql = `CREATE TABLE IF NOT EXISTS "${name}" (${columnDefs})`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
await this.sqlite3.step(prepared.stmt);
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
}
async createTable(name: string, columns: ColumnDefinition[], noDrop?: boolean) {
if (!this.connection) throw new Error('Database not connected');
if (!noDrop) {
await this.dropTableInternal(name);
}
// Map column types - SQLite doesn't have 'auto' type, use TEXT instead
const columnDefs = columns.map(col => {
const type = col.type && col.type !== 'auto' ? col.type : 'TEXT';
return `"${col.name}" ${type}`;
}).join(', ');
const sql = `CREATE TABLE IF NOT EXISTS "${name}" (${columnDefs})`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
await this.sqlite3.step(prepared.stmt);
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
}
async createIndex(indexName: string, tableName: string, columns: string[]) {
// Indexes disabled for now
return;
}
async getColumns(name: string) {
if (!this.connection) throw new Error('Database not connected');
const columns: string[] = [];
const sql = `PRAGMA table_info("${name}")`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
while (await this.sqlite3.step(prepared.stmt) === SQLite.SQLITE_ROW) {
columns.push(await this.sqlite3.column(prepared.stmt, 1)); // Column name is at index 1
}
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
return columns;
}
async count(tableName: string) {
if (!this.connection) throw new Error('Database not connected');
let count = 0;
const sql = `SELECT COUNT(*) as count FROM "${tableName}"`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
if (await this.sqlite3.step(prepared.stmt) === SQLite.SQLITE_ROW) {
count = await this.sqlite3.column(prepared.stmt, 0);
}
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
return count;
}
async addColumns(tableName: string, newColumns: string[]) {
if (!this.connection) throw new Error('Database not connected');
for (const column of newColumns) {
const sql = `ALTER TABLE "${tableName}" ADD COLUMN "${column}" TEXT`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
await this.sqlite3.step(prepared.stmt);
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
}
}
async select(statement: string, frontmatter: Record<string, unknown>) {
if (!this.connection) throw new Error('Database not connected');
if (this.isRecreating) {
console.warn('SqlocalWorkerDatabase: Database is being recreated, cannot execute select');
return { data: [], columns: [], executionTime: 0 };
}
try {
// Replace frontmatter placeholders in the query
let processedStatement = statement;
const params: any[] = [];
// Support both {{key}} and @key parameter formats for compatibility
for (const [key, value] of Object.entries(frontmatter)) {
// Handle {{key}} format (used in user queries)
const doubleBracePlaceholder = `{{${key}}}`;
const doubleBraceRegex = new RegExp(doubleBracePlaceholder.replace(/[{}]/g, '\\$&'), 'g');
const doubleBraceMatches = (processedStatement.match(doubleBraceRegex) || []).length;
if (doubleBraceMatches > 0) {
processedStatement = processedStatement.replace(doubleBraceRegex, '?');
for (let i = 0; i < doubleBraceMatches; i++) {
params.push(value);
}
}
// Handle @key format (used in repository queries)
const atPlaceholder = `@${key}`;
const atRegex = new RegExp(`@${key}\\b`, 'g');
const atMatches = (processedStatement.match(atRegex) || []).length;
if (atMatches > 0) {
processedStatement = processedStatement.replace(atRegex, '?');
for (let i = 0; i < atMatches; i++) {
params.push(value);
}
}
}
const startTime = performance.now();
const data: any[] = [];
let columns: string[] = [];
// Create string in WASM memory
const str = this.sqlite3.str_new(this.connection, processedStatement);
let prepared = null;
try {
prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared && prepared.stmt) {
await this.sqlite3.bind_collection(prepared.stmt, params);
// Get column names
const columnCount = await this.sqlite3.column_count(prepared.stmt);
for (let i = 0; i < columnCount; i++) {
columns.push(await this.sqlite3.column_name(prepared.stmt, i));
}
// Fetch all rows
let stepResult;
while ((stepResult = await this.sqlite3.step(prepared.stmt)) === SQLite.SQLITE_ROW) {
const row: any = {};
for (let i = 0; i < columnCount; i++) {
const columnName = columns[i];
row[columnName] = await this.sqlite3.column(prepared.stmt, i);
}
data.push(row);
}
}
} finally {
// Finalize statement before finishing string
if (prepared && prepared.stmt) {
try {
await this.sqlite3.finalize(prepared.stmt);
} catch (finalizeError) {
console.error('SqlocalWorkerDatabase: Error finalizing statement:', finalizeError);
}
}
this.sqlite3.str_finish(str);
}
const executionTime = performance.now() - startTime;
return {
data,
columns,
executionTime
};
} catch (error) {
console.error('SqlocalWorkerDatabase: Error executing select:', error);
console.error('SqlocalWorkerDatabase: Failed statement:', statement);
throw error;
}
}
async explain(statement: string, frontmatter: Record<string, unknown>) {
if (!this.connection) throw new Error('Database not connected');
// Replace frontmatter placeholders in the query
let processedStatement = statement;
const params: any[] = [];
// Support both {{key}} and @key parameter formats
for (const [key, value] of Object.entries(frontmatter)) {
const doubleBracePlaceholder = `{{${key}}}`;
const doubleBraceRegex = new RegExp(doubleBracePlaceholder.replace(/[{}]/g, '\\$&'), 'g');
const doubleBraceMatches = (processedStatement.match(doubleBraceRegex) || []).length;
if (doubleBraceMatches > 0) {
processedStatement = processedStatement.replace(doubleBraceRegex, '?');
for (let i = 0; i < doubleBraceMatches; i++) {
params.push(value);
}
}
const atRegex = new RegExp(`@${key}\\b`, 'g');
const atMatches = (processedStatement.match(atRegex) || []).length;
if (atMatches > 0) {
processedStatement = processedStatement.replace(atRegex, '?');
for (let i = 0; i < atMatches; i++) {
params.push(value);
}
}
}
const explainQuery = `EXPLAIN QUERY PLAN ${processedStatement}`;
const results: Array<{id: number, parent: number, detail: string}> = [];
const str = this.sqlite3.str_new(this.connection, explainQuery);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared && prepared.stmt) {
await this.sqlite3.bind_collection(prepared.stmt, params);
// EXPLAIN QUERY PLAN columns: id, parent, notused, detail
while (await this.sqlite3.step(prepared.stmt) === SQLite.SQLITE_ROW) {
const id = await this.sqlite3.column(prepared.stmt, 0);
const parent = await this.sqlite3.column(prepared.stmt, 1);
const detail = await this.sqlite3.column(prepared.stmt, 3);
results.push({ id, parent, detail });
}
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
// Format results as indented string (matching SqlocalDatabase implementation)
let strResult = '';
const map = new Map<number, number>();
const INDENT_INCREASE = 4;
map.set(0, -INDENT_INCREASE);
for (const result of results || []) {
const parent = parseInt((result.parent as unknown as string) ?? '0', 10);
const indent = (map.get(parent) || 0) + INDENT_INCREASE;
for (let i = 0; i < indent; i++) {
strResult += ' ';
}
strResult += result.detail + "\n";
map.set(result.id as number, indent);
}
return strResult;
}
async hasTable(tableName: string) {
if (!this.connection) throw new Error('Database not connected');
const sql = `SELECT name FROM sqlite_master WHERE type='table' AND name=?`;
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared) {
await this.sqlite3.bind_collection(prepared.stmt, [tableName]);
const hasRow = await this.sqlite3.step(prepared.stmt) === SQLite.SQLITE_ROW;
await this.sqlite3.finalize(prepared.stmt);
return hasRow;
}
} finally {
this.sqlite3.str_finish(str);
}
return false;
}
}
// Expose the class via Comlink so it can be used from the main thread
Comlink.expose(SqlocalWorkerDatabase);

256
src/modules/database/worker/database.ts
View file

@ -1,256 +0,0 @@
import * as Comlink from "comlink"
// @ts-ignore
import initSqlJs from '@jlongster/sql.js';
// @ts-ignore
import wasmBinary from '../../../../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm'
// @ts-ignore
import { SQLiteFS } from 'absurd-sql';
// @ts-ignore
import IndexedDBBackend from '../../../../node_modules/absurd-sql/dist/indexeddb-backend.js';
import type { BindParams, Database, Statement } from "sql.js";
import { uniq, uniqBy } from "lodash";
import { ColumnDefinition } from "../../../utils/types";
import { sanitise } from "../../../utils/sanitiseColumn";
export function toObjectArray(stmt: Statement) {
const ret = []
while (stmt.step()) {
ret.push(stmt.getAsObject())
}
return ret
}
function recordToBindParams(record: Record<string, unknown>) {
const bindParams = Object.fromEntries(Object.entries(record).map(([key, val]) => ([`@${key}`, val]))) as BindParams
return bindParams
}
const formatData = (data: Record<string, any>) => {
return Object.keys(data).reduce((ret, key) => {
if (typeof data[key] === 'boolean') {
return {
...ret,
[key]: data[key] ? 1 : 0
}
}
if (!data[key]) {
return {
...ret,
[key]: null
}
}
if (typeof data[key] === 'object' || Array.isArray(data[key])) {
return {
...ret,
[key]: JSON.stringify(data[key])
}
}
return {
...ret,
[key]: data[key]
}
}, {})
}
// Disabling SharedArrayBuffer - otherwise Absurd-SQL doesn't handle properly it.
self.SharedArrayBuffer = undefined as any
export class WorkerDatabase {
private db: Database
constructor(private readonly dbName: string) {
}
registerCustomFunction(name: string, argsCount = 1) {
const fn = (...arg: string[]) => {
const data = {
type: name,
values: arg
}
return `SQLSEALCUSTOM(${JSON.stringify(data)})`
}
// This is such a stupid solution but number of arguments needs to be static so SQLite understands which overload to use.
if (argsCount >= 1) {
this.db.create_function(name, (a: string) => fn(a))
}
if (argsCount >= 2) {
this.db.create_function(name, (a: string, b: string) => fn(a, b))
}
if (argsCount >= 3) {
this.db.create_function(name, (a: string, b: string, c: string) => fn(a, b, c))
}
if (argsCount >= 4) {
throw new Error('Too many arguments, only up to 3 arguments are supported at the moment.')
}
}
createTableText(tableName: string, fields: string[]) {
const fieldsString = fields.map(f => `${f} TEXT`).join(',')
this.db.exec(`CREATE TABLE IF NOT EXISTS ${tableName} (:fieldsString)`, {
tableName,
fieldsString
})
}
async recreateDatabase() {
this.db.run(`
PRAGMA writable_schema = 1;
DELETE FROM sqlite_master;
PRAGMA writable_schema = 0;
VACUUM;
PRAGMA integrity_check;
`)
}
run(query: string, params?: BindParams) {
this.db.run(query, params)
}
getColumns(tableName: string) {
const [data] = this.db.exec('select name from pragma_table_info(@tableName)', { '@tableName': tableName })
return data.values.map(d => d[0] as unknown as string)
}
addColumns(tableName: string, newColumns: string[]) {
for (const columnName of newColumns) {
const stmt = `ALTER TABLE ${tableName} ADD COLUMN ${columnName}`
this.db.run(stmt)
}
}
/* Types are optional in SQLite, we can take advantage of that */
async createTableNoTypes(tableName: string, columns: string[], noDrop: boolean = false) {
const fields = uniq(columns.map(f => sanitise(f)))
if (!noDrop) {
await this.dropTable(tableName)
}
const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fields.join(',')})`
this.db.run(createStmt)
}
async createTable(tableName: string, columns: ColumnDefinition[], noDrop: boolean = false) {
const fields = uniqBy(columns.map(c => ({
...c,
name: sanitise(c.name)
})), 'name')
if (!noDrop) {
await this.dropTable(tableName)
}
const fieldDefinitions = fields.map(c => c.name) // Setting type inside engine changes nothing for SQLite
const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fieldDefinitions.join(', ')})`
this.db.run(createStmt)
}
async clearTable(tableName: string) {
this.db.run(`DELETE FROM ${tableName}`)
}
async insertData(tableName: string, data: Record<string, unknown>[]) {
data.forEach(d => {
const columns = Object.keys(d).filter(c => c !== '__parsed_extra')
const insertStatement = this.db.prepare(`INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
insertStatement.run(recordToBindParams(formatData(d)))
insertStatement.free()
})
}
async dropTable(tableName: string) {
this.db.run(`DROP TABLE IF EXISTS ${tableName}`)
}
async select(query: string, params: Record<string, unknown>) {
const stmt = this.db.prepare(query, recordToBindParams(params))
const data = toObjectArray(stmt)
const columns = stmt.getColumnNames()
stmt.free()
return { data: data, columns }
}
async updateData(tableName: string, data: Array<Record<string, unknown>>, matchKey: string = 'id') {
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
data.forEach((d: Record<string, unknown>) => {
const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE ${matchKey} = @${matchKey}`)
stmt.run(recordToBindParams(formatData(d)))
stmt.free()
})
}
async deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
data.forEach(d => {
const stmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`);
stmt.run({
[`@${key}`]: d[key]
} as BindParams)
stmt.free()
})
}
async connect() {
try {
const SQL = await initSqlJs({
wasmBinary: wasmBinary
});
let sqlFS = new SQLiteFS(SQL.FS, new IndexedDBBackend(() => {
console.error('unable to write to indexedDb')
}));
SQL.register_for_idb(sqlFS);
SQL.FS.mkdir('/sql');
SQL.FS.mount(sqlFS, {}, '/sql');
const path = `sql/sqlseal___${sanitise(this.dbName)}.db`
let stream = SQL.FS.open(path, 'a+');
await stream.node.contents.readIfFallback();
SQL.FS.close(stream);
const db = new SQL.Database(path, { filename: true });
let cacheSize = 0;
let pageSize = 4096;
db.exec(`
PRAGMA cache_size=-${cacheSize};
PRAGMA journal_mode=MEMORY;
PRAGMA page_size=${pageSize};
VACUUM;
`);
this.db = db
return
} catch (e) {
console.error('Error while setting up database', e);
throw e;
}
}
async explainQuery(query: string, params: Record<string, unknown>) {
const stmt = this.db.prepare(`EXPLAIN QUERY PLAN ${query}`, recordToBindParams(params))
const plan = []
while (stmt.step()) {
plan.push(stmt.getAsObject())
}
stmt.free()
return plan
}
async disconnect() {
// FIXME: implement.
}
async createIndex(indexName: string, tableName: string, columns: string[]) {
const query = `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} (${columns.join(', ')})`
this.db.run(query)
}
}
Comlink.expose(WorkerDatabase);

4
src/modules/editor/codeblockHandler/CodeblockProcessor.ts
View file

@ -7,7 +7,7 @@ import {
import { Sync } from "../../sync/sync/sync";
import { RendererRegistry, RenderReturn } from "../renderer/rendererRegistry";
import { ParserResult, parseWithDefaults, TableDefinition } from "../parser";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { displayError, displayNotice } from "../../../utils/ui";
import { transformQuery } from "../sql/sqlTransformer";
import { registerObservers } from "../../../utils/registerObservers";
@ -26,7 +26,7 @@ export class CodeblockProcessor extends MarkdownRenderChild {
private source: string,
private ctx: MarkdownPostProcessorContext,
private rendererRegistry: RendererRegistry,
private db: Pick<SqlSealDatabase, 'select' | 'explain'>,
private db: Pick<SqlocalDatabaseProxy, 'select' | 'explain'>,
private cellParser: ModernCellParser,
private settings: Settings,
private app: App,

4
src/modules/editor/codeblockHandler/SqlSealCodeblockHandler.ts
View file

@ -1,7 +1,7 @@
import { App, MarkdownPostProcessorContext } from "obsidian"
import { RendererRegistry } from "../renderer/rendererRegistry"
import { CodeblockProcessor } from "./CodeblockProcessor"
import { SqlSealDatabase } from "../../database/database"
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy"
import { Sync } from "../../sync/sync/sync"
import { Settings } from "../../settings/Settings"
import { ModernCellParser } from "../../syntaxHighlight/cellParser/ModernCellParser"
@ -9,7 +9,7 @@ import { ModernCellParser } from "../../syntaxHighlight/cellParser/ModernCellPar
export class SqlSealCodeblockHandler {
constructor(
private readonly app: App,
private readonly db: SqlSealDatabase,
private readonly db: SqlocalDatabaseProxy,
private readonly cellParser: ModernCellParser,
private sync: Sync,
private rendererRegistry: RendererRegistry,

4
src/modules/editor/codeblockHandler/inline/InlineCodeHandler.ts
View file

@ -1,6 +1,6 @@
import { App, MarkdownPostProcessorContext, Plugin } from "obsidian";
import { InlineProcessor } from "./InlineProcessor";
import { SqlSealDatabase } from "../../../database/database";
import { SqlocalDatabaseProxy } from "../../../database/sqlocal/sqlocalDatabaseProxy";
import { Sync } from "../../../sync/sync/sync";
import { Settings } from "../../../settings/Settings";
@ -8,7 +8,7 @@ import { Settings } from "../../../settings/Settings";
export class SqlSealInlineHandler {
constructor(
private readonly app: App,
private readonly db: SqlSealDatabase,
private readonly db: SqlocalDatabaseProxy,
private readonly settings: Settings,
private sync: Sync
) { }

4
src/modules/editor/codeblockHandler/inline/InlineProcessor.ts
View file

@ -1,7 +1,7 @@
import { OmnibusRegistrator } from "@hypersphere/omnibus";
import { App, MarkdownRenderChild } from "obsidian";
import { transformQuery } from "../../sql/sqlTransformer";
import { SqlSealDatabase } from "../../../database/database";
import { SqlocalDatabaseProxy } from "../../../database/sqlocal/sqlocalDatabaseProxy";
import { Sync } from "../../../sync/sync/sync";
import { registerObservers } from "../../../../utils/registerObservers";
import { displayError } from "../../../../utils/ui";
@ -14,7 +14,7 @@ export class InlineProcessor extends MarkdownRenderChild {
private el: HTMLElement,
private query: string,
private sourcePath: string,
private db: SqlSealDatabase,
private db: SqlocalDatabaseProxy,
private settings: Settings,
private app: App,
private sync: Sync

4
src/modules/editor/init.ts
View file

@ -1,5 +1,5 @@
import { App, Plugin } from "obsidian";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
import { Sync } from "../sync/sync/sync";
import { RendererRegistry } from "./renderer/rendererRegistry";
import { TableRenderer } from "./renderer/TableRenderer";
@ -14,7 +14,7 @@ import { createSqlSealEditorExtension } from "../syntaxHighlight/editorExtension
export const editorInit = (
app: App,
db: SqlSealDatabase,
db: SqlocalDatabaseProxy,
plugin: Plugin,
sync: Sync,
inlineHandler: SqlSealInlineHandler,

4
src/modules/editor/module.ts
View file

@ -1,6 +1,6 @@
import { Registrator } from "@hypersphere/dity"
import { App, Plugin } from "obsidian"
import { SqlSealDatabase } from "../database/database"
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy"
import { Sync } from "../sync/sync/sync"
import { RendererRegistry } from "./renderer/rendererRegistry"
import { editorInit } from "./init"
@ -11,7 +11,7 @@ import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser
export const editor = new Registrator()
.import<'app', App>()
.import<'db', Promise<SqlSealDatabase>>()
.import<'db', Promise<SqlocalDatabaseProxy>>()
.import<'plugin', Plugin>()
.import<'sync', Promise<Sync>>()
.import<'cellParser', Promise<ModernCellParser>>()

4
src/modules/editor/sql/sqlTransformer.ts
View file

@ -1,4 +1,4 @@
import { uniq } from 'lodash';
import _ from 'lodash';
import { parse, show, cstVisitor } from 'sql-parser-cst';
/**
@ -33,6 +33,6 @@ export const transformQuery = (query: string, tableNames: Record<string, string>
return {
sql: show(cst),
mappedTables: uniq(watchTables)
mappedTables: _.uniq(watchTables)
}
}

4
src/modules/explorer/Editor.ts
View file

@ -3,7 +3,7 @@ import { CodeblockProcessor } from "../editor/codeblockHandler/CodeblockProcesso
import { EditorView, keymap } from "@codemirror/view";
import { ViewPluginGeneratorType } from "../syntaxHighlight/viewPluginGenerator";
import { EditorMenuBar } from "./EditorMenuBar";
import { MemoryDatabase } from "./database/memoryDatabase";
import { WaSqliteMemoryDatabase } from "./database/waSqliteMemoryDatabase";
import { SchemaVisualiser } from "./schemaVisualiser/SchemaVisualiser";
import { activateView } from "./activateView";
import { App } from "obsidian";
@ -32,7 +32,7 @@ export class Editor {
private viewPluginGenerator: ViewPluginGeneratorType,
private app: App,
private query: string = DEFAULT_QUERY,
private db: MemoryDatabase | null = null,
private db: WaSqliteMemoryDatabase | null = null,
private isTextFile: boolean = false,
private rendererRegistry?: RendererRegistry
) {

4
src/modules/explorer/InitFactory.ts
View file

@ -1,5 +1,5 @@
import { addIcon, App, Plugin } from "obsidian";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser";
import { RendererRegistry } from "../editor/renderer/rendererRegistry";
import { Sync } from "../sync/sync/sync";
@ -17,7 +17,7 @@ import SQLSealIcon from "./sqlseal-bw.svg";
export const explorerInit = (
plugin: Plugin,
app: App,
db: SqlSealDatabase,
db: SqlocalDatabaseProxy,
cellParser: ModernCellParser,
rendererRegistry: RendererRegistry,
sync: Sync,

17
src/modules/explorer/SQLSealFileView.ts
View file

@ -1,6 +1,6 @@
import { FileView, IconName, MarkdownPostProcessorContext, Menu, TextFileView, TFile, WorkspaceLeaf } from "obsidian";
import { GridApi } from "ag-grid-community";
import { MemoryDatabase } from "./database/memoryDatabase";
import { WaSqliteMemoryDatabase } from "./database/waSqliteMemoryDatabase";
import { DatabaseManager } from "./database/databaseManager";
import { TableInfo } from "./schemaVisualiser/TableVisualiser";
import { Editor } from "./Editor";
@ -10,7 +10,7 @@ import { RendererRegistry } from "../editor/renderer/rendererRegistry";
import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser";
import { Settings } from "../settings/Settings";
import { Sync } from "../sync/sync/sync";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
export const SQLSEAL_FILE_VIEW = 'sqlseal-file-view';
@ -18,7 +18,7 @@ const DEFAULT_SQLITE_QUERY = "SELECT name\nFROM sqlite_master\nWHERE type='table
const DEFAULT_SQL_QUERY = "SELECT *\nFROM files\nLIMIT 10";
export class SQLSealFileView extends TextFileView {
private fileDb: MemoryDatabase | null = null;
private fileDb: WaSqliteMemoryDatabase | null = null;
private schema: TableInfo[] = [];
private editor: Editor | null = null;
private fileContent: string = "";
@ -31,7 +31,7 @@ export class SQLSealFileView extends TextFileView {
private cellParser: ModernCellParser,
private settings: Settings,
private sync: Sync,
private vaultDb: Pick<SqlSealDatabase, 'select' | 'explain'>,
private vaultDb: Pick<SqlocalDatabaseProxy, 'select' | 'explain'>,
) {
super(leaf);
}
@ -92,7 +92,7 @@ export class SQLSealFileView extends TextFileView {
try {
this.fileDb = await this.manager.getDatabaseConnection(this.file);
await this.fileDb.connect();
this.schema = this.fileDb.getSchema();
this.schema = await this.fileDb.getSchema();
} catch (error) {
console.error("Failed to connect to database:", error);
return;
@ -118,13 +118,14 @@ export class SQLSealFileView extends TextFileView {
frontmatter: variables || {},
} as any;
// Create a database adapter to handle both MemoryDatabase and SqlSealDatabase
// Create a database adapter to handle both MemoryDatabase and SqlocalDatabaseProxy
const dbAdapter = this.fileDb ? {
select: async (statement: string, frontmatter: Record<string, unknown>) => {
const result = this.fileDb!.select(statement);
const result = await this.fileDb!.select(statement);
return {
data: result.data,
columns: Array.isArray(result.columns) ? result.columns : Object.keys(result.columns)
columns: Array.isArray(result.columns) ? result.columns : Object.keys(result.columns),
executionTime: 0
};
},
explain: async () => ""

10
src/modules/explorer/database/__tests__/__mocks__/wa-sqlite-wasm-url.ts Normal file
View file

@ -0,0 +1,10 @@
// Mock for virtual:wa-sqlite-wasm-url used in tests
// Returns a file:// URL that works with the fetch polyfill in jest.setup.mjs
import { join } from 'path';
import { pathToFileURL } from 'url';
const wasmPath = join(process.cwd(), 'node_modules/wa-sqlite/dist/wa-sqlite-async.wasm');
const wasmUrl = pathToFileURL(wasmPath).href;
export default wasmUrl;

352
src/modules/explorer/database/__tests__/memoryDatabase.test.ts Normal file
View file

@ -0,0 +1,352 @@
import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { WaSqliteMemoryDatabase } from '../waSqliteMemoryDatabase';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// Mock TFile from Obsidian
class MockTFile {
name: string;
path: string;
vault: any;
constructor(path: string, data: ArrayBuffer) {
this.name = path.split('/').pop() || 'test.db';
this.path = path;
this.vault = {
readBinary: async (file: any) => data
};
}
}
describe('WaSqliteMemoryDatabase with wa-sqlite', () => {
let memoryDb: WaSqliteMemoryDatabase;
let testDbBuffer: ArrayBuffer;
let mockFile: any;
beforeAll(() => {
// Load test database
const testDbPath = join(__dirname, 'test.db');
testDbBuffer = readFileSync(testDbPath).buffer;
});
beforeEach(() => {
// Create new mock file and WaSqliteMemoryDatabase instance for each test
mockFile = new MockTFile('test.db', testDbBuffer);
memoryDb = new WaSqliteMemoryDatabase(mockFile);
});
afterEach(async () => {
if (memoryDb) {
await memoryDb.disconnect();
}
});
describe('Basic Query Execution', () => {
test('should execute simple SELECT query', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync<{ result: number }>('SELECT 1 as result');
expect(results.columns).toEqual(['result']);
expect(results.data).toEqual([{ result: 1 }]);
});
test('should return empty array for queries with no results', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync('SELECT * FROM users WHERE id = 999');
expect(results.data).toEqual([]);
});
test('should execute SELECT * FROM users', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync<{ id: number, name: string, age: number }>('SELECT * FROM users ORDER BY id');
expect(results.columns).toEqual(['id', 'name', 'age']);
expect(results.data).toEqual([
{ id: 1, name: 'Alice', age: 30 },
{ id: 2, name: 'Bob', age: 25 }
]);
});
});
describe('Schema Queries', () => {
test('should query sqlite_master for tables', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync<{ name: string }>("SELECT name FROM sqlite_master WHERE type='table'");
expect(results.columns).toEqual(['name']);
expect(results.data).toContainEqual({ name: 'users' });
});
test('should use PRAGMA table_info', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync<{ name: string, type: string }>("SELECT name, type FROM pragma_table_info('users')");
expect(results.columns).toEqual(['name', 'type']);
expect(results.data).toContainEqual({ name: 'id', type: 'INTEGER' });
expect(results.data).toContainEqual({ name: 'name', type: 'TEXT' });
expect(results.data).toContainEqual({ name: 'age', type: 'INTEGER' });
});
});
describe('Parameter Binding', () => {
test('should handle parameterized queries', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync<{ id: number, name: string, age: number }>('SELECT * FROM users WHERE id = ?', { id: 1 });
expect(results.data).toEqual([{ id: 1, name: 'Alice', age: 30 }]);
});
test('should handle empty parameter object', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync('SELECT * FROM users', {});
expect(results.data).toHaveLength(2);
});
test('should handle no parameters (null)', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync('SELECT * FROM users', null);
expect(results.data).toHaveLength(2);
});
test('should handle no parameters (undefined)', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync('SELECT * FROM users');
expect(results.data).toHaveLength(2);
});
});
describe('Data Transformation', () => {
test('should transform rows to objects', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync<{ id: number, name: string, age: number }>('SELECT * FROM users ORDER BY id');
expect(results.data).toEqual([
{ id: 1, name: 'Alice', age: 30 },
{ id: 2, name: 'Bob', age: 25 }
]);
});
test('should handle empty results when transforming', async () => {
await memoryDb.connect();
const results = await memoryDb.queryAsync('SELECT * FROM users WHERE id = 999');
expect(results.data).toEqual([]);
// Columns should still be returned even when there are no rows
expect(results.columns).toEqual(['id', 'name', 'age']);
});
});
describe('Realistic MemoryDatabase method calls', () => {
test('should call allTables() successfully', async () => {
await memoryDb.connect();
const results = await memoryDb.allTables();
expect(results.data.length).toBeGreaterThan(0);
expect(results.data).toContainEqual({ name: 'users' });
});
test('should call getColumns() successfully', async () => {
await memoryDb.connect();
const results = await memoryDb.getColumns('users');
expect(results.data.length).toBe(3);
expect(results.data).toContainEqual(expect.objectContaining({ name: 'id' }));
expect(results.data).toContainEqual(expect.objectContaining({ name: 'name' }));
expect(results.data).toContainEqual(expect.objectContaining({ name: 'age' }));
});
test('REPRODUCE BUG: Call queryAsync the same way allTables() does', async () => {
await memoryDb.connect();
// This mimics the exact call pattern from allTables()
const results = await memoryDb.queryAsync<{ name: string }>("SELECT name FROM sqlite_master WHERE type='table'");
expect(results.data.length).toBeGreaterThan(0);
expect(results.data).toContainEqual({ name: 'users' });
});
test('REPRODUCE BUG: Call queryAsync with explicit null', async () => {
await memoryDb.connect();
// This mimics passing null explicitly
const results = await memoryDb.queryAsync<{ name: string }>("SELECT name FROM sqlite_master WHERE type='table'", null);
expect(results.data.length).toBeGreaterThan(0);
expect(results.data).toContainEqual({ name: 'users' });
});
test('REPRODUCE BUG: Call queryAsync with undefined', async () => {
await memoryDb.connect();
// This might be what's causing the error
const results = await memoryDb.queryAsync<{ name: string }>("SELECT name FROM sqlite_master WHERE type='table'", undefined as any);
expect(results.data.length).toBeGreaterThan(0);
expect(results.data).toContainEqual({ name: 'users' });
});
});
describe('All MemoryDatabase methods', () => {
beforeEach(async () => {
await memoryDb.connect();
});
test('select() method should work with parameters', async () => {
const results = await memoryDb.select<{ id: number, name: string }>('SELECT id, name FROM users WHERE id = ?', { id: 1 });
expect(results.data).toEqual([{ id: 1, name: 'Alice' }]);
});
test('select() method should work without parameters', async () => {
const results = await memoryDb.select('SELECT * FROM users ORDER BY id');
expect(results.data).toHaveLength(2);
});
test.skip('query() method (synchronous) not supported in wa-sqlite', async () => {
// wa-sqlite is async-only, synchronous query() is not supported
// This test is skipped for the new wa-sqlite implementation
expect(() => {
memoryDb.query('SELECT * FROM users ORDER BY id');
}).toThrow('Synchronous query() not supported');
});
test('getAllTables() should return list of table names', async () => {
const tables = await memoryDb.getAllTables();
expect(Array.isArray(tables)).toBe(true);
expect(tables).toContain('users');
});
test('getDetailedTableInfo() should return detailed column information', async () => {
const info = await memoryDb.getDetailedTableInfo('users');
expect(info).toHaveLength(3);
const idColumn = info.find(col => col.name === 'id');
expect(idColumn).toBeDefined();
expect(idColumn?.isPrimaryKey).toBe(true);
expect(idColumn?.type).toBe('INTEGER');
const nameColumn = info.find(col => col.name === 'name');
expect(nameColumn).toBeDefined();
expect(nameColumn?.notNull).toBe(true);
expect(nameColumn?.type).toBe('TEXT');
});
test('getForeignKeys() should return foreign key information', async () => {
const fks = await memoryDb.getForeignKeys('users');
// Test database has no foreign keys
expect(Array.isArray(fks)).toBe(true);
expect(fks).toHaveLength(0);
});
test('getDetailedSchema() should return full schema for all tables', async () => {
const schema = await memoryDb.getDetailedSchema();
expect(Array.isArray(schema)).toBe(true);
expect(schema.length).toBeGreaterThan(0);
const usersTable = schema.find(table => table.name === 'users');
expect(usersTable).toBeDefined();
expect(usersTable?.columns).toHaveLength(3);
expect(usersTable?.foreignKeys).toBeDefined();
});
test('getSchema() should return TableInfo array', async () => {
const schema = await memoryDb.getSchema();
expect(Array.isArray(schema)).toBe(true);
expect(schema.length).toBeGreaterThan(0);
const usersTable = schema.find(table => table.name === 'users');
expect(usersTable).toBeDefined();
expect(usersTable?.name).toBe('users');
expect(usersTable?.columns).toHaveLength(3);
});
test('explain() should return empty object', () => {
const result = memoryDb.explain();
expect(result).toEqual({});
});
test('REPRODUCE BUG: Calling select() without await should return a Promise', async () => {
// This is what's happening in SQLSealFileView.ts line 124
const result = memoryDb.select('SELECT * FROM users');
// Result should be a Promise
expect(result).toBeInstanceOf(Promise);
// Trying to access .data or .columns on a Promise would fail
expect((result as any).data).toBeUndefined();
expect((result as any).columns).toBeUndefined();
// This would cause "cannot convert undefined" errors
// Clean up: await the promise to finalize the statement
await result;
});
});
describe('Error handling', () => {
test('should throw error when querying before connect()', () => {
expect(() => {
memoryDb.query('SELECT * FROM users');
}).toThrow('Synchronous query() not supported');
});
test('should handle invalid SQL gracefully', async () => {
await memoryDb.connect();
await expect(async () => {
await memoryDb.queryAsync('INVALID SQL STATEMENT');
}).rejects.toThrow();
});
test('should handle queries on non-existent tables', async () => {
await memoryDb.connect();
await expect(async () => {
await memoryDb.queryAsync('SELECT * FROM nonexistent_table');
}).rejects.toThrow();
});
test('should handle invalid table names in getColumns()', async () => {
await memoryDb.connect();
const results = await memoryDb.getColumns('nonexistent_table');
expect(results.data).toEqual([]);
});
});
describe('Disconnect behavior', () => {
test('should close database connection', async () => {
await memoryDb.connect();
const resultBefore = await memoryDb.queryAsync('SELECT 1 as test');
expect(resultBefore.data).toEqual([{ test: 1 }]);
await memoryDb.disconnect();
await expect(async () => {
await memoryDb.queryAsync('SELECT 1 as test');
}).rejects.toThrow('Database not connected');
});
test('should be safe to disconnect multiple times', async () => {
await memoryDb.connect();
await memoryDb.disconnect();
await memoryDb.disconnect(); // Should not throw
await expect(async () => {
await memoryDb.queryAsync('SELECT 1');
}).rejects.toThrow('Database not connected');
});
});
});

BIN
src/modules/explorer/database/__tests__/test.db Normal file
View file

Binary file not shown.

36
src/modules/explorer/database/databaseManager.ts
View file

@ -1,34 +1,20 @@
import { TFile } from "obsidian";
import { MemoryDatabase } from "./memoryDatabase";
// @ts-ignore
import wasmBinary from '../../../../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm'
// @ts-ignore
import initSqlJs from '@jlongster/sql.js';
import { WaSqliteMemoryDatabase } from "./waSqliteMemoryDatabase";
/**
* DatabaseManager - manages connections to external .db files
* Using wa-sqlite with MemoryAsyncVFS for in-memory database loading
*/
export class DatabaseManager {
constructor() {}
private sql: initSqlJs.SqlJsStatic | null = null
private async getConnection() {
if (this.sql) {
return this.sql
}
const SQL = await initSqlJs({
wasmBinary: wasmBinary,
});
this.sql = SQL
return SQL
}
async getDatabaseConnection(file: TFile) {
// FIXME: connecting to database
const connection = await this.getConnection()
const db = new MemoryDatabase(connection, file);
await db.connect()
return db
const db = new WaSqliteMemoryDatabase(file);
await db.connect();
return db;
}
getGlobalDatabaseConnection() {}
getGlobalDatabaseConnection() {
throw new Error('Global database connection not implemented. Use the main database provider instead.');
}
}

119
src/modules/explorer/database/memoryDatabase.ts
View file

@ -1,119 +0,0 @@
import { TFile } from "obsidian";
import { BindParams, Database, ParamsObject } from "sql.js";
import { toObjectArray } from "../../database/worker/database";
import { TableInfo } from "../schemaVisualiser/TableVisualiser";
export class MemoryDatabase {
private db: Database
constructor(private sql: initSqlJs.SqlJsStatic, private file: TFile) {
}
async connect() {
const binary = await this.file.vault.readBinary(this.file)
this.db = new this.sql.Database(Buffer.from(binary))
}
query<T = ParamsObject>(query: string, params: BindParams = null): { data: T[], columns: keyof T } {
const stmt = this.db.prepare(query, params)
const data = toObjectArray(stmt)
const columns = stmt.getColumnNames()
stmt.free()
return { data: data, columns } as any
}
select<T = ParamsObject>(query: string, params: BindParams = null) {
return this.query<T>(query, params)
}
explain() {
return {}
}
allTables() {
return this.query<{name: string}>(`select name from sqlite_master where type='table'`)
}
getColumns(tableName: string) {
return this.query<{ name: string, type: string }>('select name, type from pragma_table_info(@tableName)', { '@tableName': tableName })
}
getDetailedTableInfo(tableName: string) {
const result = this.query<{
name: string,
type: string,
pk: number,
dflt_value: any,
notnull: number
}>(`
SELECT name, type, pk, dflt_value, [notnull]
FROM pragma_table_info(@tableName)
`, { '@tableName': tableName })
return result.data.map(row => ({
name: row.name,
type: row.type,
isPrimaryKey: row.pk === 1,
defaultValue: row.dflt_value,
notNull: row.notnull === 1
}))
}
getForeignKeys(tableName: string) {
const result = this.query<{
id: number,
seq: number,
table: string,
from: string,
to: string,
on_update: string,
on_delete: string,
match: string
}>(`
SELECT id, seq, [table], [from], [to], on_update, on_delete, [match]
FROM pragma_foreign_key_list(@tableName)
`, { '@tableName': tableName })
return result.data.map(row => ({
id: row.id,
seq: row.seq,
referencedTable: row.table,
fromColumn: row.from,
toColumn: row.to,
onUpdate: row.on_update,
onDelete: row.on_delete,
match: row.match
}))
}
getAllTables() {
const result = this.query<{ name: string }>(`
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
ORDER BY name
`)
return result.data.map(row => row.name)
}
getDetailedSchema() {
const tables = this.getAllTables()
return tables.map(tableName => ({
name: tableName,
columns: this.getDetailedTableInfo(tableName),
foreignKeys: this.getForeignKeys(tableName)
}))
}
getSchema(): TableInfo[] {
const tables = this.allTables().data
return tables.map(t => {
const columns = this.getColumns(t.name)
return {
name: t.name,
columns: columns.data
}
})
}
}

246
src/modules/explorer/database/waSqliteMemoryDatabase.ts Normal file
View file

@ -0,0 +1,246 @@
import { TFile } from "obsidian";
import { TableInfo } from "../schemaVisualiser/TableVisualiser";
import SQLiteAsyncESMFactory from 'wa-sqlite/dist/wa-sqlite-async.mjs';
import * as SQLite from 'wa-sqlite';
import { MemoryAsyncVFS } from 'wa-sqlite/src/examples/MemoryAsyncVFS.js';
// @ts-ignore - Virtual module from esbuild
import wasmUrl from 'virtual:wa-sqlite-wasm-url';
type ParamsObject = Record<string, any>;
/**
* WaSqliteMemoryDatabase - reads external .db files using wa-sqlite
* This is used by the SQL Explorer to open and query external SQLite database files
*
* Uses wa-sqlite with MemoryAsyncVFS for simple, direct Uint8Array → database loading
*/
export class WaSqliteMemoryDatabase {
private connection: number | null = null;
private sqlite3: any = null;
private vfs: MemoryAsyncVFS | null = null;
private readonly dbName = 'external.db';
constructor(private file: TFile) {}
async connect() {
// Read the .db file binary data
const binary = await this.file.vault.readBinary(this.file);
const data = new Uint8Array(binary);
// Validate SQLite database
const header = new TextDecoder().decode(data.slice(0, 16));
if (!header.startsWith('SQLite format 3')) {
throw new Error('Invalid SQLite database file format');
}
// Initialize wa-sqlite
const asyncModule = await SQLiteAsyncESMFactory({
locateFile: (file: string) => {
if (file.endsWith('.wasm')) {
return wasmUrl;
}
return file;
}
});
this.sqlite3 = SQLite.Factory(asyncModule);
// Create and register MemoryAsyncVFS
this.vfs = new MemoryAsyncVFS();
this.sqlite3.vfs_register(this.vfs, true); // true = make it the default VFS
// Pre-populate the VFS with the database file data
// The MemoryVFS stores files in a Map keyed by filename
this.vfs.mapNameToFile.set(this.dbName, {
name: this.dbName,
flags: 0,
size: data.byteLength,
data: data.buffer // Use the ArrayBuffer from the Uint8Array
});
// Open the database connection
this.connection = await this.sqlite3.open_v2(
this.dbName,
SQLite.SQLITE_OPEN_READONLY // Open as read-only since we're just querying
);
}
private async runQuery<T = ParamsObject>(sql: string, params: any[] = []): Promise<{ data: T[], columns: string[] }> {
if (!this.connection || !this.sqlite3) {
throw new Error('Database not connected');
}
try {
const data: T[] = [];
let columns: string[] = [];
// Create string in WASM memory
const str = this.sqlite3.str_new(this.connection, sql);
try {
const prepared = await this.sqlite3.prepare_v2(this.connection, this.sqlite3.str_value(str));
if (prepared && prepared.stmt) {
// Bind parameters if any
if (params.length > 0) {
await this.sqlite3.bind_collection(prepared.stmt, params);
}
// Get column names
const columnCount = await this.sqlite3.column_count(prepared.stmt);
for (let i = 0; i < columnCount; i++) {
columns.push(await this.sqlite3.column_name(prepared.stmt, i));
}
// Fetch all rows
while (await this.sqlite3.step(prepared.stmt) === SQLite.SQLITE_ROW) {
const row: any = {};
for (let i = 0; i < columnCount; i++) {
row[columns[i]] = await this.sqlite3.column(prepared.stmt, i);
}
data.push(row as T);
}
await this.sqlite3.finalize(prepared.stmt);
}
} finally {
this.sqlite3.str_finish(str);
}
return { data, columns };
} catch (error) {
console.error('WaSqliteMemoryDatabase: Query execution failed', { sql, params, error });
throw error;
}
}
query<T = ParamsObject>(query: string, params: Record<string, any> | null = null): { data: T[], columns: string[] } {
// Convert params object to array for wa-sqlite
const paramArray = params && typeof params === 'object' && !Array.isArray(params) ? Object.values(params) : [];
// This is a sync method in the original API, but we need async for wa-sqlite
// We'll need to make this async-compatible
throw new Error('Synchronous query() not supported in wa-sqlite implementation. Use queryAsync() instead.');
}
async queryAsync<T = ParamsObject>(query: string, params: Record<string, any> | null = null): Promise<{ data: T[], columns: string[] }> {
const paramArray = params && typeof params === 'object' && !Array.isArray(params) ? Object.values(params) : [];
return this.runQuery<T>(query, paramArray);
}
async select<T = ParamsObject>(query: string, params: Record<string, any> | null = null) {
return this.queryAsync<T>(query, params);
}
explain() {
return {};
}
async allTables() {
return this.queryAsync<{name: string}>(`SELECT name FROM sqlite_master WHERE type='table'`);
}
async getColumns(tableName: string) {
return this.queryAsync<{ name: string, type: string }>(
`SELECT name, type FROM pragma_table_info('${tableName}')`,
null
);
}
async getDetailedTableInfo(tableName: string) {
const result = await this.queryAsync<{
name: string,
type: string,
pk: number,
dflt_value: any,
notnull: number
}>(`
SELECT name, type, pk, dflt_value, [notnull]
FROM pragma_table_info('${tableName}')
`, null);
return result.data.map(row => ({
name: row.name,
type: row.type,
isPrimaryKey: row.pk === 1,
defaultValue: row.dflt_value,
notNull: row.notnull === 1
}));
}
async getForeignKeys(tableName: string) {
const result = await this.queryAsync<{
id: number,
seq: number,
table: string,
from: string,
to: string,
on_update: string,
on_delete: string,
match: string
}>(`
SELECT id, seq, [table], [from], [to], on_update, on_delete, [match]
FROM pragma_foreign_key_list('${tableName}')
`, null);
return result.data.map(row => ({
id: row.id,
seq: row.seq,
referencedTable: row.table,
fromColumn: row.from,
toColumn: row.to,
onUpdate: row.on_update,
onDelete: row.on_delete,
match: row.match
}));
}
async getAllTables() {
const result = await this.queryAsync<{ name: string }>(`
SELECT name FROM sqlite_master
WHERE type='table' AND name NOT LIKE 'sqlite_%'
ORDER BY name
`);
return result.data.map(row => row.name);
}
async getDetailedSchema() {
const tables = await this.getAllTables();
const schema = [];
for (const tableName of tables) {
schema.push({
name: tableName,
columns: await this.getDetailedTableInfo(tableName),
foreignKeys: await this.getForeignKeys(tableName)
});
}
return schema;
}
async getSchema(): Promise<TableInfo[]> {
const tablesResult = await this.allTables();
const tables = tablesResult.data;
const schema: TableInfo[] = [];
for (const t of tables) {
const columns = await this.getColumns(t.name);
schema.push({
name: t.name,
columns: columns.data
});
}
return schema;
}
async disconnect() {
if (this.connection && this.sqlite3) {
await this.sqlite3.close(this.connection);
this.connection = null;
}
// VFS cleanup is automatic when connection closes
this.vfs = null;
this.sqlite3 = null;
}
}

4
src/modules/explorer/explorer/ExplorerView.ts
View file

@ -6,7 +6,7 @@ import {
WorkspaceLeaf,
} from "obsidian";
import { CodeblockProcessor } from "../../editor/codeblockHandler/CodeblockProcessor";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { RendererRegistry } from "../../editor/renderer/rendererRegistry";
import { Settings } from "../../settings/Settings";
import { ModernCellParser } from "../../syntaxHighlight/cellParser/ModernCellParser";
@ -19,7 +19,7 @@ export class ExplorerView extends ItemView {
constructor(
leaf: WorkspaceLeaf,
private rendererRegistry: RendererRegistry,
private db: Pick<SqlSealDatabase, 'select' | 'explain'>,
private db: Pick<SqlocalDatabaseProxy, 'select' | 'explain'>,
private cellParser: ModernCellParser,
private settings: Settings,
private sync: Sync,

4
src/modules/explorer/module.ts
View file

@ -2,7 +2,7 @@ import { Registrator } from "@hypersphere/dity";
import { explorerInit } from "./InitFactory";
import { App, Plugin } from "obsidian";
import { ModernCellParser } from "../syntaxHighlight/cellParser/ModernCellParser";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
import { Settings } from "../settings/Settings";
import { Sync } from "../sync/sync/sync";
import { RendererRegistry } from "../editor/renderer/rendererRegistry";
@ -13,7 +13,7 @@ import { DatabaseManager } from "./database/databaseManager";
export const explorer = new Registrator()
.import<'app', App>()
.import<'cellParser', Promise<ModernCellParser>>()
.import<'db', Promise<SqlSealDatabase>>()
.import<'db', Promise<SqlocalDatabaseProxy>>()
.import<'settings', Promise<Settings>>()
.import<'sync', Promise<Sync>>()
.import<'rendererRegistry', RendererRegistry>()

21
src/modules/explorer/schemaVisualiser/SchemaVisualiser.ts
View file

@ -1,5 +1,5 @@
import mermaid from 'mermaid'
import { MemoryDatabase } from '../database/memoryDatabase'
import { WaSqliteMemoryDatabase } from '../database/waSqliteMemoryDatabase'
export interface DetailedColumnInfo {
name: string
@ -29,7 +29,7 @@ export interface DetailedTableInfo {
export class SchemaVisualiser {
private initialized = false
constructor(private database: MemoryDatabase) {
constructor(private database: WaSqliteMemoryDatabase) {
this.initializeMermaid()
}
@ -67,14 +67,7 @@ export class SchemaVisualiser {
try {
const schema = await this.buildSchema()
const mermaidCode = this.generateMermaidERD(schema)
// Debug: Log the generated Mermaid code
console.log('Generated Mermaid ERD Code:', mermaidCode)
console.log('Schema tables detected:', schema.map(t => ({
original: t.name,
escaped: this.escapeMermaidIdentifier(t.name)
})))
const diagramContainer = container.createDiv({
cls: 'sqlseal-mermaid-container',
attr: {
@ -107,14 +100,10 @@ export class SchemaVisualiser {
this.addInteractivity(diagramContainer, schema)
// Remove relationship summary - user requested removal
} catch (error) {
console.error('Error rendering Mermaid schema:', error)
console.error('Error details:', {
message: error instanceof Error ? error.message : 'Unknown error',
stack: error instanceof Error ? error.stack : undefined
})
// Fallback to simple schema display
this.showFallbackSchema(container, await this.buildSchema())
}

3
src/modules/main/init.ts
View file

@ -19,5 +19,8 @@ export const mainInit = (
apiInit();
globalTablesInit();
explorerInit();
console.log('🚀 SQL Seal initialized with wa-sqlite test command available');
console.log('📋 Use Ctrl/Cmd+P -> "Test wa-sqlite Implementation" to test wa-sqlite');
};
};

2
src/modules/main/module.ts
View file

@ -19,7 +19,7 @@ const obsidian = new Registrator({ logger: console.log })
.export('app', 'plugin', 'vault')
export const mainModule = new Registrator()
export const mainModule = new Registrator({logger: console.log})
.module('obsidian', obsidian)
.module('db', db)
.module('editor', editor)

6
src/modules/sync/fileSyncController/fileSyncFactory.ts
View file

@ -1,7 +1,7 @@
import { App, Plugin } from "obsidian";
import { SealFileSync } from "./FileSync";
import { Sync } from "../sync/sync";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { FilesFileSyncTable } from "../sync/tables/filesTable";
import { TagsFileSyncTable } from "../sync/tables/tagsTable";
import { TasksFileSyncTable } from "../sync/tables/tasksTable";
@ -10,10 +10,12 @@ import { LinksFileSyncTable } from "../sync/tables/linksTable";
export const fileSyncFactory = async (
app: App,
plugin: Plugin,
db: SqlSealDatabase,
dbPromise: Promise<SqlocalDatabaseProxy>,
sync: Sync,
) => {
return async () => {
const db = await dbPromise;
const fileSync = new SealFileSync(app, plugin, (name) =>
sync.triggerGlobalTableChange(name),
);

8
src/modules/sync/module.ts
View file

@ -1,16 +1,20 @@
import { Registrator } from "@hypersphere/dity"
import { App, Plugin, Vault } from "obsidian"
import { syncBusFactory } from "./sync/syncFactory"
import { SqlSealDatabase } from "../database/database"
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy"
import { syncInit } from "./sync/init"
import { fileSyncFactory } from "./fileSyncController/fileSyncFactory"
export const sync = new Registrator()
.import<'app', App>()
.import<'db', Promise<SqlocalDatabaseProxy>>()
.import<'plugin', Plugin>()
.import<'db', Promise<SqlSealDatabase>>()
.import<'vault', Vault>()
// @ts-expect-error - TypeScript has trouble inferring Registrator types after SqlocalDatabaseProxy changes
.register('syncBus', d => d.fn(syncBusFactory).inject('db', 'vault', 'app'))
// @ts-expect-error - TypeScript has trouble inferring Registrator types after SqlocalDatabaseProxy changes
.register('fileSync', d => d.fn(fileSyncFactory).inject('app', 'plugin', 'db', 'syncBus'))
.register('init', d => d.fn(syncInit).inject('app', 'fileSync'))
.export('init', 'syncBus')
export type SyncModule = typeof sync

5
src/modules/sync/repository/abstractRepository.ts
View file

@ -1,5 +1,6 @@
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
export abstract class Repository {
constructor(protected readonly db: SqlSealDatabase) { }
constructor(protected readonly db: SqlocalDatabaseProxy) {
}
}

2
src/modules/sync/repository/tableDefinitions.ts
View file

@ -98,7 +98,7 @@ export class TableDefinitionsRepository extends Repository {
updateData.arguments = JSON.stringify(fields.arguments);
}
await this.db.db!.updateData(this.TABLE_NAME, [{
await this.db.updateData(this.TABLE_NAME, [{
id,
...updateData
}], 'id')

79
src/modules/sync/sync/sync.ts
View file

@ -2,7 +2,7 @@ import { App, TAbstractFile, TFile, Vault } from "obsidian";
import { FilepathHasher } from "../../../utils/hasher";
import { Omnibus } from "@hypersphere/omnibus";
import { uniq } from "lodash";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { TableDefinition, TableDefinitionsRepository } from "../repository/tableDefinitions";
import { TableAliasesRepository } from "../repository/tableAliases";
import { ConfigurationRepository } from "../repository/configuration";
@ -12,17 +12,22 @@ import { SyncStrategyFactory } from "../syncStrategy/SyncStrategyFactory";
const SQLSEAL_DATABASE_VERSION = 2;
// Global lock to prevent concurrent database recreation
let isInitializing = false;
let initializationPromise: Promise<void> | null = null;
export class Sync {
private tableDefinitionsRepo: TableDefinitionsRepository;
private tableMapLog: TableAliasesRepository;
private bus = new Omnibus()
private configRepo: ConfigurationRepository;
private bus = new Omnibus();
private isLocallyInitialized = false;
constructor(
private readonly db: SqlSealDatabase,
private readonly db: SqlocalDatabaseProxy,
private readonly vault: Vault,
private readonly app: App
) {
}
triggerGlobalTableChange(name: string) {
@ -30,13 +35,43 @@ export class Sync {
}
async init() {
const instanceId = Math.random().toString(36).substring(7);
// If this instance is already initialized, don't do it again
if (this.isLocallyInitialized) {
return;
}
// If another init is already in progress, wait for it
if (isInitializing && initializationPromise) {
await initializationPromise;
// Just set up the local repositories, don't recreate database
await this.setupRepositories(instanceId);
return;
}
// Start initialization process
isInitializing = true;
initializationPromise = this.performInitialization(instanceId);
try {
await initializationPromise;
} finally {
isInitializing = false;
initializationPromise = null;
}
}
private async performInitialization(instanceId: string) {
await this.db.connect()
// Configuration
const config = new ConfigurationRepository(this.db)
this.configRepo = new ConfigurationRepository(this.db)
let version
try {
version = await config.getConfig('version') as number
version = await this.configRepo.getConfig('version') as number
} catch (e) {
version = 0
}
@ -45,7 +80,7 @@ export class Sync {
await this.db.recreateDatabase()
}
await config.init()
await this.configRepo.init()
this.tableDefinitionsRepo = new TableDefinitionsRepository(this.db)
await this.tableDefinitionsRepo.init()
@ -53,7 +88,7 @@ export class Sync {
this.tableMapLog = new TableAliasesRepository(this.db)
await this.tableMapLog.init()
await config.setConfig('version', SQLSEAL_DATABASE_VERSION)
await this.configRepo.setConfig('version', SQLSEAL_DATABASE_VERSION)
const fileLogs = await this.tableDefinitionsRepo.getAll()
const uniquePaths = uniq(fileLogs.map(l => l.source_file))
@ -65,6 +100,25 @@ export class Sync {
this.startSync()
await this.refreshGlobalMappings()
this.isLocallyInitialized = true;
}
private async setupRepositories(instanceId: string) {
await this.db.connect()
// Create repository instances but don't call init() since tables are already created
this.configRepo = new ConfigurationRepository(this.db)
this.tableDefinitionsRepo = new TableDefinitionsRepository(this.db)
this.tableMapLog = new TableAliasesRepository(this.db)
// START SYNCING
this.startSync()
// Don't call refreshGlobalMappings() - it will be called by the primary initialization
// await this.refreshGlobalMappings()
this.isLocallyInitialized = true;
}
async syncFileByName(fileName: string) {
@ -128,20 +182,21 @@ export class Sync {
async getTablesMappingForContext(sourceFileName: string) {
const tables = await this.tableMapLog.getByContext(sourceFileName) as { alias_name: string, table_name: string }[]
const map = tables.reduce((acc, t) => ({
...acc,
[t.alias_name as string]: t.table_name
}), {})
// FIXME: adding globals here.
return {
const result = {
...map,
...this.globalTablesMapping,
files: 'files',
tasks: 'tasks',
tags: 'tags',
}
};
return result;
}
async generateTableName(fileName: string) {

11
src/modules/sync/sync/syncFactory.ts
View file

@ -1,9 +1,14 @@
import { App, Vault } from "obsidian";
import { Sync } from "./sync";
import { SqlSealDatabase } from "../../database/database";
export const syncBusFactory = async (db: SqlSealDatabase, vault: Vault, app: App) => {
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
let singletonSync: Sync|null = null
export const syncBusFactory = async (dbPromise: Promise<SqlocalDatabaseProxy>, vault: Vault, app: App) => {
if (singletonSync) {
return singletonSync
}
const db = await dbPromise;
const sync = new Sync(db, vault, app);
singletonSync = sync
await sync.init();
return sync;
};

4
src/modules/sync/sync/tables/abstractFileSyncTable.ts
View file

@ -1,9 +1,9 @@
import { App, TFile } from "obsidian";
import { SqlSealDatabase } from "../../../database/database";
import { SqlocalDatabaseProxy } from "../../../database/sqlocal/sqlocalDatabaseProxy";
export abstract class AFileSyncTable {
constructor(
protected readonly db: SqlSealDatabase,
protected readonly db: SqlocalDatabaseProxy,
protected readonly app: App
) {
}

6
src/modules/sync/sync/tables/filesTable.ts
View file

@ -2,7 +2,7 @@ import { App, Plugin, TFile } from "obsidian";
import { AFileSyncTable } from "./abstractFileSyncTable";
import { difference } from "lodash";
import { sanitise } from "../../../../utils/sanitiseColumn";
import { SqlSealDatabase } from "../../../database/database";
import { SqlocalDatabaseProxy } from "../../../database/sqlocal/sqlocalDatabaseProxy";
export const FILES_TABLE_NAME = 'files'
@ -35,7 +35,7 @@ export class FilesFileSyncTable extends AFileSyncTable {
}
private columns: string[] = []
shouldPerformBulkInsert = true;
constructor(db: SqlSealDatabase, app: App, private readonly plugin: Plugin) {
constructor(db: SqlocalDatabaseProxy, app: App, private readonly plugin: Plugin) {
super(db, app)
}
async onFileModify(file: TFile): Promise<void> {
@ -80,7 +80,7 @@ export class FilesFileSyncTable extends AFileSyncTable {
async onInit(): Promise<void> {
this.db.createTableNoTypes(FILES_TABLE_NAME, ['id', 'name', 'path', 'created_at', 'modified_at', 'file_size'])
await this.db.createTableNoTypes(FILES_TABLE_NAME, ['id', 'name', 'path', 'created_at', 'modified_at', 'file_size'])
this.columns = (await this.db.getColumns(FILES_TABLE_NAME)) ?? []
// Indexes

5
src/modules/syntaxHighlight/cellParser/ModernCellParser.ts
View file

@ -1,5 +1,5 @@
import { isStringifiedArray } from "../../../utils/ui";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { CellFunction } from "./CellFunction";
import { parse } from 'json5'
@ -172,7 +172,8 @@ export class ModernCellParser {
}
// FIXME: this should be extracted to separate class / function but for now it's fine.
registerDbFunctions(db: SqlSealDatabase) {
registerDbFunctions(db: SqlocalDatabaseProxy) {
console.trace('register db functions called')
this.functions.forEach(funct => {
db.registerCustomFunction(funct.name, funct.sqlFunctionArgumentsCount)
})

4
src/modules/syntaxHighlight/cellParser/factory.ts
View file

@ -3,7 +3,7 @@ import { ModernCellParser } from "./ModernCellParser";
import { LinkParser } from "./parser/link";
import { ImageParser } from "./parser/image";
import { CheckboxParser } from "./parser/checkbox";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
export const getCellParser = (app: App, create = createEl) => {
const cellParser = new ModernCellParser();
@ -15,7 +15,7 @@ export const getCellParser = (app: App, create = createEl) => {
export const cellParserFactory = (
app: App,
db: SqlSealDatabase,
db: SqlocalDatabaseProxy,
create: typeof createEl = createEl,
) => {
const cellParser = new ModernCellParser();

4
src/modules/syntaxHighlight/editorExtension/inlineCodeBlock.ts
View file

@ -9,7 +9,7 @@ import {
import { RangeSetBuilder } from "@codemirror/state";
import { syntaxTree } from "@codemirror/language";
import { App } from "obsidian";
import { SqlSealDatabase } from "../../database/database";
import { SqlocalDatabaseProxy } from "../../database/sqlocal/sqlocalDatabaseProxy";
import { Settings } from "../../settings/Settings";
import { Sync } from "../../sync/sync/sync";
import { SqlSealInlineHandler } from "../../editor/codeblockHandler/inline/InlineCodeHandler";
@ -17,7 +17,7 @@ import { InlineProcessor } from "../../editor/codeblockHandler/inline/InlineProc
export function createSqlSealEditorExtension(
app: App,
db: SqlSealDatabase,
db: SqlocalDatabaseProxy,
settings: Settings,
sync: Sync,
) {

4
src/modules/syntaxHighlight/module.ts
View file

@ -3,13 +3,13 @@ import { syntaxHighlightInit } from "./init";
import { App, Plugin } from "obsidian";
import { RendererRegistry } from "../editor/renderer/rendererRegistry";
import { cellParserFactory } from "./cellParser/factory";
import { SqlSealDatabase } from "../database/database";
import { SqlocalDatabaseProxy } from "../database/sqlocal/sqlocalDatabaseProxy";
import { viewPluginGeneratorFactory } from "./viewPluginGenerator";
export const syntaxHighlight = new Registrator()
.import<'app', App>()
.import<'db', Promise<SqlSealDatabase>>()
.import<'db', Promise<SqlocalDatabaseProxy>>()
.import<'rendererRegistry', RendererRegistry>()
.import<'plugin', Plugin>()
.register('cellParser', d => d.fn(cellParserFactory).inject('app', 'db'))

15
src/types/wa-sqlite.d.ts vendored Normal file
View file

@ -0,0 +1,15 @@
// Type declarations for wa-sqlite modules without TypeScript support
declare module 'wa-sqlite/src/examples/IDBBatchAtomicVFS.js' {
export interface VFSOptions {
durability?: "default" | "strict" | "relaxed";
purge?: "deferred" | "manual";
purgeAtLeast?: number;
}
export class IDBBatchAtomicVFS {
constructor(idbDatabaseName?: string, options?: VFSOptions);
close(): Promise<void>;
name: string;
}
}

1
src/utils/registerObservers.test.ts
View file

@ -1,3 +1,4 @@
import { jest, describe, it, expect } from '@jest/globals';
import { Omnibus } from "@hypersphere/omnibus"
import { registerObservers } from "./registerObservers"

2
src/utils/sanitiseColumn.ts
View file

@ -1,4 +1,4 @@
const unidecode = require('unidecode')
import unidecode from 'unidecode';
/**
* Sanitizes a string to be used as a valid SQLite column name.

1
tsconfig.json
View file

@ -12,6 +12,7 @@
"isolatedModules": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"strict": false,
"types": [
"jest"
],