obsidian-plugin/h-sphere_sql-seal
1
0
Fork 0
mirror of https://github.com/h-sphere/sql-seal.git synced 2026-07-22 10:10:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: code cleanup. simplifying code and making less data reloads. fixed database connections and no longer loading all the files into memeory at startup

Browse source
This commit is contained in:
Kacper Kula 2025-01-21 19:36:37 +00:00
parent 58af2197d3
commit 3267c8cfc5
16 changed files with 91 additions and 292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
src/database/database.ts
View file

@ -1,41 +1,8 @@
import { App } from "obsidian"
import { FieldTypes, toTypeStatements } from "../utils/typePredictions"
import * as Comlink from 'comlink'
import workerCode from 'virtual:worker-code'
import { WorkerDatabase } from "./worker/database";
export interface FieldDefinition {
name: string;
type: FieldTypes
}
const formatData = (data: Record<string, any>) => {
return Object.keys(data).reduce((ret, key) => {
if (typeof data[key] === 'boolean') {
return {
...ret,
[key]: data[key] ? 1 : 0
}
}
if (!data[key]) {
return {
...ret,
[key]: null
}
}
if (typeof data[key] === 'object' || Array.isArray(data[key])) {
return {
...ret,
[key]: JSON.stringify(data[key])
}
}
return {
...ret,
[key]: data[key]
}
}, {})
}
export class SqlSealDatabase {
db: Comlink.Remote<WorkerDatabase>
private isConnected = false
@ -88,14 +55,6 @@ export class SqlSealDatabase {
await this.db.recreateDatabase()
}
async createTableWithData(name: string, data: Array<Record<string, unknown>>) {
const schema = await this.getSchema(data)
await this.createTable(name, schema)
await this.insertData(name, data)
return schema
}
async updateData(name: string, data: Array<Record<string, unknown>>) {
return this.db.updateData(name, data)
}
@ -112,26 +71,16 @@ export class SqlSealDatabase {
return this.db.dropTable(name)
}
async createTableClean(name: string, fields: Array<FieldDefinition>) {
const fieldsToRecord = fields.reduce((acc, f) => ({
...acc,
[f.name]: f.type
}), {} as Record<string, FieldTypes>) as Record<string, FieldTypes>
await this.createTable(name, fieldsToRecord)
}
async createTable(name: string, fields: Record<string, FieldTypes>, noDrop?: boolean) {
await this.db.createTable(name, fields, noDrop)
}
async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) {
await this.db.createTableNoTypes(name, columns, noDrop)
}
async getSchema(data: Array<Record<string, unknown>>) {
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
const { types } = toTypeStatements(fields, data as Array<Record<string, string>>); // Call the toTypeStatements function with the correct arguments
return types;
async getColumns(tableName: string) {
return this.db.getColumns(tableName)
}
async addColumns(tableName: string, newColumns: string[]) {
return this.db.addColumns(tableName, newColumns)
}
async select(statement: string, frontmatter: Record<string, unknown>) {

47
src/database/worker/database.ts
View file

@ -5,7 +5,6 @@ import { SQLiteFS } from 'absurd-sql';
import IndexedDBBackend from '../../../node_modules/absurd-sql/dist/indexeddb-backend.js';
import type { BindParams, Database, Statement } from "sql.js";
import { sanitise } from "../../utils/sanitiseColumn";
import { FieldTypes } from "../../utils/typePredictions";
import { uniq } from "lodash";
@ -114,7 +113,7 @@ export class WorkerDatabase {
}
async recreateDatabase() {
this.db.exec(`
this.db.run(`
PRAGMA writable_schema = 1;
DELETE FROM sqlite_master;
PRAGMA writable_schema = 0;
@ -123,24 +122,20 @@ export class WorkerDatabase {
`)
}
async createTable(tableName: string, fields: Record<string, FieldTypes>, noDrop: boolean = false) {
const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
const uniqueFields = [...new Map(transformedFiels.map(item =>
[item[0], item])).values()]
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)
// FIXME: probably use schema generator, for now create with hardcoded fields
if (!noDrop) {
await this.dropTable(tableName)
}
const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} (
${sqlFields.join(', ')}
);`
run(query: string, params?: BindParams) {
this.db.run(query, params)
}
this.db.prepare(createSQL).run()
if (!noDrop) {
await this.clearTable(tableName)
getColumns(tableName: string) {
const [data] = this.db.exec('select name from pragma_table_info(@tableName)', { '@tableName': tableName })
return data.values.map(d => d[0] as unknown as string)
}
addColumns(tableName: string, newColumns: string[]) {
for (const columnName of newColumns) {
const stmt = `ALTER TABLE ${tableName} ADD COLUMN ${columnName}`
this.db.run(stmt)
}
// Dropping data.
}
/* Types are optional in SQLite, we can take advantage of that */
@ -150,11 +145,11 @@ export class WorkerDatabase {
await this.dropTable(tableName)
}
const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fields.join(',')})`
this.db.prepare(createStmt).run()
this.db.run(createStmt)
}
async clearTable(tableName: string) {
this.db.prepare(`DELETE FROM ${tableName}`).run()
this.db.run(`DELETE FROM ${tableName}`)
}
@ -163,6 +158,7 @@ export class WorkerDatabase {
const columns = Object.keys(d).filter(c => c !== '__parsed_extra')
const insertStatement = this.db.prepare(`INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
insertStatement.run(recordToBindParams(formatData(d)))
insertStatement.free()
})
}
@ -172,10 +168,11 @@ export class WorkerDatabase {
async select(query: string, params: Record<string, unknown>) {
const stmt = this.db.prepare(query, recordToBindParams(params))
return {
data: toObjectArray(stmt),
columns: stmt.getColumnNames()
}
const data = toObjectArray(stmt)
const columns = stmt.getColumnNames()
stmt.free()
return { data, columns }
}
async updateData(tableName: string, data: Array<Record<string, unknown>>, matchKey: string = 'id') {
@ -183,6 +180,7 @@ export class WorkerDatabase {
data.forEach((d: Record<string, unknown>) => {
const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE ${matchKey} = @${matchKey}`)
stmt.run(recordToBindParams(d))
stmt.free()
})
}
@ -192,6 +190,7 @@ export class WorkerDatabase {
stmt.run({
[`@${key}`]: d[key]
} as BindParams)
stmt.free()
})
}

11
src/datamodel/repository/fileLog.ts
View file

@ -20,16 +20,7 @@ export class FileLogRepository extends Repository {
}
private async createTable() {
await this.db.createTable('file_log', {
'id': 'TEXT',
'table_name': 'TEXT',
'file_name': 'TEXT',
'created_at': 'TEXT',
'updated_at': 'TEXT',
'file_hash': 'TEXT',
'type': 'TEXT',
extras: 'TEXT'
}, true)
await this.db.createTableNoTypes('file_log', ['id', 'table_name', 'file_name','created_at','updated_at','file_hash','type','extras'], true)
}
async insert(log: Omit<FileLog, 'id' | 'created_at' | 'updated_at'>) {

11
src/datamodel/repository/tableMapLog.ts
View file

@ -22,14 +22,9 @@ export class TableMapLogRepository extends Repository {
private async createTable() {
// FIXME: add autoincrement index here to make it easier to manage.
await this.db.createTable('table_map_log', {
'id': 'TEXT',
'table_name': 'TEXT',
'alias_name': 'TEXT',
'source_file_name': 'TEXT',
'created_at': 'TEXT',
'updated_at': 'TEXT'
}, true)
await this.db.createTableNoTypes('table_map_log',[
'id', 'table_name', 'alias_name', 'source_file_name', 'created_at', 'updated_at'
], true)
}
async insert(log: Omit<TableMapLog, 'id' | 'created_at' | 'updated_at'>) {

2
src/datamodel/sync.ts
View file

@ -93,7 +93,7 @@ export class Sync {
const { data, columns } = await syncObject.returnData()
const tableName = entry.table_name
await this.db.createTableClean(tableName, columns)
await this.db.createTableNoTypes(tableName, columns)
await this.db.insertData(tableName, data)
await this.fileLog.updateHash(tableName, file.stat.mtime.toString())
this.bus.trigger('change::' + tableName)

16
src/datamodel/syncStrategy/CsvFileSyncStrategy.ts
View file

@ -3,7 +3,6 @@ import { TableRegistration } from "../types";
import { ISyncStrategy } from "./abstractSyncStrategy";
import { sanitise } from "../../utils/sanitiseColumn";
import { parse } from "papaparse";
import { FieldTypes, toTypeStatements } from "../../utils/typePredictions";
import { FilepathHasher } from "../../utils/hasher";
import { TableDefinitionConfig } from "./types";
import { SourceType } from "../../grammar/newParser";
@ -29,19 +28,20 @@ export class CsvFileSyncStrategy implements ISyncStrategy {
async returnData() {
const file = this.app.vault.getFileByPath(this.reg.sourceFile)!
const data = await this.app.vault.cachedRead(file)
// TODO: PROBABLY SHOULD BE EXTRACTED SOMEWHERE FROM HERE later.
const parsed = parse<Record<string, string>>(data, {
header: true,
dynamicTyping: false,
dynamicTyping: true,
skipEmptyLines: true,
transformHeader: sanitise
})
const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
name: key,
type: value as FieldTypes
}));
// const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
// const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
// name: key,
// type: value as FieldTypes
// }));
return { columns, data: parsed.data }
return { data: parsed.data, columns: parsed.meta.fields ?? [] }
}
}

13
src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts
View file

@ -3,7 +3,6 @@ import { TableRegistration } from "../types";
import { ISyncStrategy } from "./abstractSyncStrategy";
import type { App } from "obsidian";
import { Component, MarkdownRenderer } from "obsidian";
import { FieldTypes, toTypeStatements } from "../../utils/typePredictions";
import { TableDefinitionConfig } from "./types";
import { SourceType } from "../../grammar/newParser";
@ -54,7 +53,7 @@ export class MarkdownTableSyncStrategy implements ISyncStrategy {
// Check if the requested table exists
if (tableIndex >= tables.length) {
return null;
return { data: [], columns: [] };
}
const targetTable = tables[tableIndex];
@ -75,15 +74,9 @@ export class MarkdownTableSyncStrategy implements ISyncStrategy {
}, {} as Record<string, string>);
});
const typeStatements = toTypeStatements(headers ?? [], data)
const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
name: key,
type: value as FieldTypes
}));
return {
columns,
data
data,
columns: headers ?? []
};
}
}

5
src/datamodel/syncStrategy/abstractSyncStrategy.ts
View file

@ -1,4 +1,7 @@
export interface ISyncStrategy {
tableName(): Promise<string>;
returnData(): any;
returnData(): Promise<{
data: Record<string, unknown>[],
columns: string[]
}>;
}

1
src/editorExtension/inlineCodeBlock.ts
View file

@ -3,7 +3,6 @@ import { RangeSetBuilder } from "@codemirror/state";
import { syntaxTree } from "@codemirror/language";
import { App } from "obsidian";
import { SqlSealDatabase } from "../database/database";
import { RendererRegistry } from "../renderer/rendererRegistry";
import { Sync } from "../datamodel/sync";
import { SqlSealInlineHandler } from "src/codeblockHandler/inline/InlineCodeHandler";
import { InlineProcessor } from "src/codeblockHandler/inline/InlineProcessor";

17
src/utils/typePredictions.test.ts
View file

@ -1,17 +0,0 @@
import { predictJson, predictType } from './typePredictions'
describe('Utils', () => {
it('should properly predict TEXT type', () => {
expect(predictType('xxx', [{ 'xxx': 'dhjkafhkjafhkfas' }])).toEqual('TEXT')
})
it('should properly predict JSON type', () => {
expect(predictType('xxx', [{ 'xxx': { a: 'b' } }])).toEqual('JSON')
})
it('should properly predict json by parsing it', () => {
expect(predictJson([{ 'xxx': '["a", "b"]'}])).toEqual([{
xxx: ['a', 'b']
}])
})
})

99
src/utils/typePredictions.ts
View file

@ -1,99 +0,0 @@
import { parse } from 'json5'
import { camelCase } from "lodash";
export const predictType = (field: string, data: Array<Record<string, string | Object>>) => {
if (field === 'id') {
return 'TEXT'
}
const d = data.map(d => d[field])
// CHECK JSONS.
const isObject = d.reduce((acc, d) => acc && (!d || typeof d === 'object'), true)
if (isObject) {
return 'JSON'
}
const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field] as string), true)
if (canBeNumber) {
// Check if Integer or Float
const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field] as string) === parseInt(d[field] as string), true)
if (canBeInteger) {
return 'INTEGER'
}
return 'REAL'
}
return 'TEXT'
}
export type FieldTypes = ReturnType<typeof predictType>
export const predictJson = (data: Array<Record<string, unknown>>) => {
return data.map(d => Object.keys(d).reduce((o, k) => {
if (typeof d[k] !== 'string') {
return {
...o,
[k]: d[k]
}
}
const v = (d[k] as string).trim()
if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) {
try {
const d = parse(v)
return {
...o,
[k]: d
}
} catch (e) {
}
}
return {
...o,
[k]: d[k]
}
}, {}))
}
export function isNumeric(str: string) {
if (typeof str != "string") return false // we only process strings!
return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
export function dataToCamelCase(data: Array<Record<string, unknown>>) {
return data.map(d => Object.keys(d).reduce((acc, k) => ({
...acc,
[camelCase(k)]: d[k]
}), {}))
}
export const toTypeStatements = (header: Array<string>, data: Array<Record<string, string>>) => {
let d: Array<Record<string, string | number>> = data
const types: Record<string, ReturnType<typeof predictType>> = {}
header.forEach(key => {
const type = predictType(key, data)
if (type === 'REAL' || type === 'INTEGER') {
// converting all data here to text
d = d.map(record => ({
...record,
[key]: type === 'REAL'
? parseFloat(record[key] as string)
: parseInt(record[key] as string)
}))
}
types[key] = type
})
return {
data: d,
types
}
}

2
src/utils/ui.ts
View file

@ -46,7 +46,7 @@ const generateLink = (config: SqlSealAnchorElement, app: App) => {
if (fileHref.endsWith('.md')) {
fileHref = fileHref.slice(0, -3)
}
const vaultName = encodeURIComponent(app.appId);
const vaultName = encodeURIComponent((app as any).appId);
const encodedPath = encodeURIComponent(fileHref);
href = `obsidian://open?vault=${vaultName}&file=${encodedPath}`;
}

27
src/vaultSync/SealFileSync.ts
View file

@ -1,5 +1,4 @@
import { App, Plugin, TFile } from "obsidian";
import { FieldTypes } from "../utils/typePredictions";
import { sanitise } from "../utils/sanitiseColumn";
import { AFileSyncTable } from "./tables/abstractFileSyncTable";
@ -13,7 +12,6 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
}
export class SealFileSync {
private currentSchema: Record<string, FieldTypes> = {}
private tablePlugins: Array<AFileSyncTable> = []
constructor(
public readonly app: App,
@ -23,14 +21,7 @@ export class SealFileSync {
if (!(file instanceof TFile)) {
return
}
await sleep(100)
const frontmatter = await extractFrontmatterFromFile(file, plugin)
if (this.hasNewColumns(frontmatter)) {
await sleep(1000)
await this.init()
return
}
await sleep(100) // FIXME: check if this is actually needed.
await Promise.all(this.tablePlugins.map(p => p.onFileModify(file)))
@ -40,15 +31,6 @@ export class SealFileSync {
if (!(file instanceof TFile)) {
return
}
const frontmatter = await extractFrontmatterFromFile(file, plugin)
if (this.hasNewColumns(frontmatter)) {
await sleep(1000)
await this.init()
return
}
await Promise.all(this.tablePlugins.map(p => p.onFileCreate(file)))
}))
@ -101,11 +83,4 @@ export class SealFileSync {
)
}
hasNewColumns(newFrontmatter: Record<string, any>) {
const currentFields = Object.keys(this.currentSchema)
const newFields = Object.keys(newFrontmatter).filter(f => !currentFields.includes(f))
return newFields.length > 0
}
}

58
src/vaultSync/tables/filesTable.ts
View file

@ -2,10 +2,13 @@ import { App, Plugin, TAbstractFile, TFile } from "obsidian";
import { AFileSyncTable } from "./abstractFileSyncTable";
import { sanitise } from "../../utils/sanitiseColumn";
import { SqlSealDatabase } from "../../database/database";
import { predictType } from "../../utils/typePredictions";
import { difference } from "lodash";
const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
export const FILES_TABLE_NAME = 'files'
const extractFrontmatterFromFile = (file: TFile, plugin: Plugin): Record<string, unknown> => {
const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
return Object.fromEntries(
@ -14,7 +17,7 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
)
}
function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<string, any>) {
function fileData(file: TAbstractFile, { tags: _tags, ...frontmatter }: Record<string, any>) {
return {
...frontmatter,
path: file.path,
@ -24,41 +27,56 @@ function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<st
}
export class FilesFileSyncTable extends AFileSyncTable {
private currentSchema: Record<string, ReturnType<typeof predictType>>
private columns: string[] = []
shouldPerformBulkInsert = true;
constructor(db: SqlSealDatabase, app: App, private readonly plugin: Plugin) {
super(db, app)
}
async onFileModify(file: TFile): Promise<void> {
// we need to update the row
const frontmatter = await extractFrontmatterFromFile(file, this.plugin)
await this.db.updateData('files', [fileData(file, frontmatter)])
const frontmatter = extractFrontmatterFromFile(file, this.plugin)
const frontmatterWithFileData = fileData(file, frontmatter)
const columns = Object.keys(frontmatterWithFileData)
await this.updateColumnsIfNeeded(columns)
await this.db.updateData(FILES_TABLE_NAME, [])
await sleep(1000) // TO DELAY OTHER PLUGINS, UGLY BUT WORKS.
}
async onFileDelete(path: string): Promise<void> {
await this.db.deleteData('files', [{ id: path }])
await this.db.deleteData(FILES_TABLE_NAME, [{ id: path }])
}
async updateColumnsIfNeeded(newSetOfColumns: string[]) {
const newColumns = difference(newSetOfColumns, this.columns)
if (newColumns.length) {
console.log('ADDING COLUMNS', newColumns)
await this.db.addColumns(FILES_TABLE_NAME, newColumns)
this.columns = await this.db.getColumns(FILES_TABLE_NAME)
}
}
async onFileCreate(file: TFile): Promise<void> {
// we need to update the row
const frontmatter = await extractFrontmatterFromFile(file, this.plugin)
// TODO: Check if there are new columns. If there are, we update the database.
await this.db.insertData('files', [fileData(file, frontmatter)])
// FIXME: should we be waiting here?
await sleep(1000)
console.log('--- PROCESSING', file.name)
const frontmatter = extractFrontmatterFromFile(file, this.plugin)
const frontmatterWithFileData = fileData(file, frontmatter)
const columns = Object.keys(frontmatterWithFileData)
await this.updateColumnsIfNeeded(columns)
await this.db.insertData(FILES_TABLE_NAME, [frontmatterWithFileData])
}
async onFileCreateBulk(files: Array<TFile>) {
// FIXME: implement this one and replace the other implementation.
const data = await Promise.all(files.map(async f => fileData(f, await extractFrontmatterFromFile(f, this.plugin))))
const schema = await this.db.createTableWithData('files', data)
this.currentSchema = schema
this.columns = await this.db.getColumns(FILES_TABLE_NAME)
// One by one
for (const file of files) {
await this.onFileCreate(file)
}
}
async onInit(): Promise<void> {
this.db.createTableNoTypes(FILES_TABLE_NAME, ['id', 'name', 'path'], true)
this.columns = await this.db.getColumns(FILES_TABLE_NAME)
}
}

5
src/vaultSync/tables/tagsTable.ts
View file

@ -32,9 +32,6 @@ export class TagsFileSyncTable extends AFileSyncTable {
async onInit(): Promise<void> {
await this.db.createTable('tags', {
'tag': 'TEXT',
'fileId': 'TEXT'
})
await this.db.createTableNoTypes('tags', ['tag', 'fileId'])
}
}

6
src/vaultSync/tables/tasksTable.ts
View file

@ -45,10 +45,6 @@ export class TasksFileSyncTable extends AFileSyncTable {
}
async onInit(): Promise<void> {
await this.db.createTable('tasks', {
'task': 'TEXT',
'completed': 'INTEGER',
'filePath': 'TEXT'
})
await this.db.createTableNoTypes('tasks', ['task', 'completed', 'filePath'])
}
}