obsidian-plugin/h-sphere_sql-seal
1
0
Fork 0
mirror of https://github.com/h-sphere/sql-seal.git synced 2026-07-22 10:10:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into feat/json-querying

Browse source
This commit is contained in:
Kacper Kula 2025-01-23 14:09:57 +00:00
commit 35eaec2c83
27 changed files with 444 additions and 631 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -1,3 +1,7 @@
# 0.18.0 (2025-01-23)
Synchronisation code has been greatly refactored. Thanks to that many common issue with files not being refreshed or pointing to the wrong file occasionally should be fixed.
Also fixed many instances of potential memory leaks. Now plugin should be more stable and reliable.
# 0.17.0 (2025-01-20)
Adding support for `LIST` view. You can now set custom classes for `HTML` (table) view.

2
manifest.json
View file

@ -1,7 +1,7 @@
{
"id": "sqlseal",
"name": "SQLSeal",
"version": "0.17.0",
"version": "0.18.0",
"minAppVersion": "0.15.0",
"description": "Use SQL in your notes to query your vault files and CSV content.",
"author": "hypersphere",

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "sqlseal",
"version": "0.17.0",
"version": "0.18.0",
"description": "A plugin for Obsidian that allows you to run SQL queries on your notes.",
"main": "main.js",
"scripts": {

31
src/codeblockHandler/CodeblockProcessor.ts
View file

@ -2,7 +2,8 @@ import { OmnibusRegistrator } from "@hypersphere/omnibus";
import { App, MarkdownPostProcessorContext, MarkdownRenderChild, TFile } from "obsidian";
import { SqlSealDatabase } from "src/database/database";
import { Sync } from "src/datamodel/sync";
import { parseLanguage, Table } from "src/grammar/newParser";
import { ParserTableDefinition } from "src/datamodel/syncStrategy/types";
import { parseLanguage } from "src/grammar/newParser";
import { RendererRegistry, RenderReturn } from "src/renderer/rendererRegistry";
import { transformQuery } from "src/sql/sqlTransformer";
import { registerObservers } from "src/utils/registerObservers";
@ -33,7 +34,7 @@ export class CodeblockProcessor extends MarkdownRenderChild {
if (results.tables) {
await this.registerTables(results.tables)
if (!results.queryPart) {
displayNotice(this.el, `Creating tables: ${results.tables.map(t => t.tableName).join(', ')}`)
displayNotice(this.el, `Creating tables: ${results.tables.map(t => t.tableAlias).join(', ')}`)
return
}
}
@ -80,29 +81,7 @@ export class CodeblockProcessor extends MarkdownRenderChild {
}
}
async registerTables(tables: Table[]) {
await Promise.all(tables.map((table) => {
let path: TFile | null = null
let extras = {}
if (table.type !== 'table') {
path = this.app.metadataCache.getFirstLinkpathDest(table.fileName, this.ctx.sourcePath)
} else {
path = this.app.vault.getFileByPath(this.ctx.sourcePath)
extras = table?.extras ?? {}
}
if (!path) {
throw new Error(`File does not exist: ${table.fileName} (for ${table.tableName}).`)
}
return this.sync.registerTable({
aliasName: table.tableName,
type: table.type,
fileName: path.path,
sourceFile: this.ctx.sourcePath,
extras
})
}))
async registerTables(tables: ParserTableDefinition[]) {
await Promise.all(tables.map((table) => this.sync.registerTable(table)))
}
}

63
src/database/database.ts
View file

@ -1,41 +1,8 @@
import { App } from "obsidian"
import { FieldTypes, toTypeStatements } from "../utils/typePredictions"
import * as Comlink from 'comlink'
import workerCode from 'virtual:worker-code'
import { WorkerDatabase } from "./worker/database";
export interface FieldDefinition {
name: string;
type: FieldTypes
}
const formatData = (data: Record<string, any>) => {
return Object.keys(data).reduce((ret, key) => {
if (typeof data[key] === 'boolean') {
return {
...ret,
[key]: data[key] ? 1 : 0
}
}
if (!data[key]) {
return {
...ret,
[key]: null
}
}
if (typeof data[key] === 'object' || Array.isArray(data[key])) {
return {
...ret,
[key]: JSON.stringify(data[key])
}
}
return {
...ret,
[key]: data[key]
}
}, {})
}
export class SqlSealDatabase {
db: Comlink.Remote<WorkerDatabase>
private isConnected = false
@ -88,14 +55,6 @@ export class SqlSealDatabase {
await this.db.recreateDatabase()
}
async createTableWithData(name: string, data: Array<Record<string, unknown>>) {
const schema = await this.getSchema(data)
await this.createTable(name, schema)
await this.insertData(name, data)
return schema
}
async updateData(name: string, data: Array<Record<string, unknown>>) {
return this.db.updateData(name, data)
}
@ -112,26 +71,16 @@ export class SqlSealDatabase {
return this.db.dropTable(name)
}
async createTableClean(name: string, fields: Array<FieldDefinition>) {
const fieldsToRecord = fields.reduce((acc, f) => ({
...acc,
[f.name]: f.type
}), {} as Record<string, FieldTypes>) as Record<string, FieldTypes>
await this.createTable(name, fieldsToRecord)
}
async createTable(name: string, fields: Record<string, FieldTypes>, noDrop?: boolean) {
await this.db.createTable(name, fields, noDrop)
}
async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) {
await this.db.createTableNoTypes(name, columns, noDrop)
}
async getSchema(data: Array<Record<string, unknown>>) {
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
const { types } = toTypeStatements(fields, data as Array<Record<string, string>>); // Call the toTypeStatements function with the correct arguments
return types;
async getColumns(tableName: string) {
return this.db.getColumns(tableName)
}
async addColumns(tableName: string, newColumns: string[]) {
return this.db.addColumns(tableName, newColumns)
}
async select(statement: string, frontmatter: Record<string, unknown>) {

47
src/database/worker/database.ts
View file

@ -5,7 +5,6 @@ import { SQLiteFS } from 'absurd-sql';
import IndexedDBBackend from '../../../node_modules/absurd-sql/dist/indexeddb-backend.js';
import type { BindParams, Database, Statement } from "sql.js";
import { sanitise } from "../../utils/sanitiseColumn";
import { FieldTypes } from "../../utils/typePredictions";
import { uniq } from "lodash";
@ -114,7 +113,7 @@ export class WorkerDatabase {
}
async recreateDatabase() {
this.db.exec(`
this.db.run(`
PRAGMA writable_schema = 1;
DELETE FROM sqlite_master;
PRAGMA writable_schema = 0;
@ -123,24 +122,20 @@ export class WorkerDatabase {
`)
}
async createTable(tableName: string, fields: Record<string, FieldTypes>, noDrop: boolean = false) {
const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
const uniqueFields = [...new Map(transformedFiels.map(item =>
[item[0], item])).values()]
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)
// FIXME: probably use schema generator, for now create with hardcoded fields
if (!noDrop) {
await this.dropTable(tableName)
}
const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} (
${sqlFields.join(', ')}
);`
run(query: string, params?: BindParams) {
this.db.run(query, params)
}
this.db.prepare(createSQL).run()
if (!noDrop) {
await this.clearTable(tableName)
getColumns(tableName: string) {
const [data] = this.db.exec('select name from pragma_table_info(@tableName)', { '@tableName': tableName })
return data.values.map(d => d[0] as unknown as string)
}
addColumns(tableName: string, newColumns: string[]) {
for (const columnName of newColumns) {
const stmt = `ALTER TABLE ${tableName} ADD COLUMN ${columnName}`
this.db.run(stmt)
}
// Dropping data.
}
/* Types are optional in SQLite, we can take advantage of that */
@ -150,11 +145,11 @@ export class WorkerDatabase {
await this.dropTable(tableName)
}
const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fields.join(',')})`
this.db.prepare(createStmt).run()
this.db.run(createStmt)
}
async clearTable(tableName: string) {
this.db.prepare(`DELETE FROM ${tableName}`).run()
this.db.run(`DELETE FROM ${tableName}`)
}
@ -163,6 +158,7 @@ export class WorkerDatabase {
const columns = Object.keys(d).filter(c => c !== '__parsed_extra')
const insertStatement = this.db.prepare(`INSERT INTO ${tableName} (${columns.join(', ')}) VALUES (${columns.map((key: string) => '@' + key).join(', ')})`);
insertStatement.run(recordToBindParams(formatData(d)))
insertStatement.free()
})
}
@ -172,10 +168,11 @@ export class WorkerDatabase {
async select(query: string, params: Record<string, unknown>) {
const stmt = this.db.prepare(query, recordToBindParams(params))
return {
data: toObjectArray(stmt),
columns: stmt.getColumnNames()
}
const data = toObjectArray(stmt)
const columns = stmt.getColumnNames()
stmt.free()
return { data, columns }
}
async updateData(tableName: string, data: Array<Record<string, unknown>>, matchKey: string = 'id') {
@ -183,6 +180,7 @@ export class WorkerDatabase {
data.forEach((d: Record<string, unknown>) => {
const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE ${matchKey} = @${matchKey}`)
stmt.run(recordToBindParams(d))
stmt.free()
})
}
@ -192,6 +190,7 @@ export class WorkerDatabase {
stmt.run({
[`@${key}`]: d[key]
} as BindParams)
stmt.free()
})
}

81
src/datamodel/repository/fileLog.ts
View file

@ -1,81 +0,0 @@
import { SourceType } from "src/grammar/newParser";
import { Repository } from "./abstractRepository";
import { v4 as uuidv4 } from "uuid";
export interface FileLog {
id: string,
table_name: string,
file_name: string,
created_at: string,
updated_at: string,
file_hash: string,
type: SourceType,
extras: Record<string, string>
}
export class FileLogRepository extends Repository {
async init() {
await this.createTable()
}
private async createTable() {
await this.db.createTable('file_log', {
'id': 'TEXT',
'table_name': 'TEXT',
'file_name': 'TEXT',
'created_at': 'TEXT',
'updated_at': 'TEXT',
'file_hash': 'TEXT',
'type': 'TEXT',
extras: 'TEXT'
}, true)
}
async insert(log: Omit<FileLog, 'id' | 'created_at' | 'updated_at'>) {
const now = Date.now()
await this.db.insertData('file_log', [{
id: uuidv4(),
table_name: log.table_name,
file_name: log.file_name,
file_hash: log.file_hash,
type: log.type,
extras: JSON.stringify(log.extras ?? {}),
created_at: now,
updated_at: now
}])
}
async getByFilename(filename: string) {
const { data } = await this.db.select('SELECT * FROM file_log WHERE file_name = @filename', {
filename: filename
})
if (!data.length) {
return null
}
const d = data[0]
return {
...d,
type: d.type ?? 'file',
extras: JSON.parse(d.extras ? d.extras.toString() : '{}')
} as FileLog
}
async getAll() {
const { data } = await this.db.select('SELECT * FROM file_log', {})
return data.map(d => ({
...d,
type: d.type ?? 'file',
extras: JSON.parse(d.extras ? d.extras.toString() : '{}')
})) as FileLog[]
}
async updateHash(tableName: string, newHash: string) {
await this.db.db.updateData('file_log', [{
table_name: tableName,
file_hash: newHash
}], 'table_name')
}
}

37
src/datamodel/repository/tableMapLog.ts → src/datamodel/repository/tableAliases.ts
View file

@ -1,7 +1,7 @@
import { Repository } from "./abstractRepository";
import { v4 as uuid4 } from "uuid";
export interface TableMapLog {
export interface TableAlias {
id: string;
table_name: string;
alias_name: string;
@ -10,31 +10,28 @@ export interface TableMapLog {
updated_at: string;
}
export class TableMapLogRepository extends Repository {
export class TableAliasesRepository extends Repository {
private readonly TABLE_NAME = 'TABLE_ALIASES';
async init() {
await this.createTable()
}
async deleteMapping(id: string) {
this.db.deleteData('table_map_log', [{ id: id }], 'id')
this.db.deleteData(this.TABLE_NAME, [{ id: id }], 'id')
}
private async createTable() {
// FIXME: add autoincrement index here to make it easier to manage.
await this.db.createTable('table_map_log', {
'id': 'TEXT',
'table_name': 'TEXT',
'alias_name': 'TEXT',
'source_file_name': 'TEXT',
'created_at': 'TEXT',
'updated_at': 'TEXT'
}, true)
await this.db.createTableNoTypes(this.TABLE_NAME,[
'id', 'table_name', 'alias_name', 'source_file_name', 'created_at', 'updated_at'
], true)
}
async insert(log: Omit<TableMapLog, 'id' | 'created_at' | 'updated_at'>) {
async insert(log: Omit<TableAlias, 'id' | 'created_at' | 'updated_at'>) {
const now = Date.now()
await this.db.insertData('table_map_log', [{
await this.db.insertData(this.TABLE_NAME, [{
id: uuid4(),
table_name: log.table_name,
alias_name: log.alias_name,
@ -45,12 +42,12 @@ export class TableMapLogRepository extends Repository {
}
async getAll() {
const { data } = await this.db.select('SELECT * FROM table_map_log', {})
return data as unknown as TableMapLog[]
const { data } = await this.db.select('SELECT * FROM TABLE_ALIASES', {})
return data as unknown as TableAlias[]
}
async getByAlias(sourceFileName: string, aliasName: string) {
const { data } = await this.db.select(`SELECT * FROM table_map_log
const { data } = await this.db.select(`SELECT * FROM TABLE_ALIASES
WHERE source_file_name=@source_file_name
AND alias_name=@alias_name`, {
'source_file_name': sourceFileName,
@ -59,23 +56,23 @@ export class TableMapLogRepository extends Repository {
if (!data || data.length < 0) {
return null
}
return data[0] as unknown as TableMapLog
return data[0] as unknown as TableAlias
}
async getByTableName(tableName: string) {
const { data } = await this.db.select(`SELECT * FROM table_map_log
const { data } = await this.db.select(`SELECT * FROM TABLE_ALIASES
WHERE table_name = @table_name`, {
table_name: tableName
})
if (!data) {
return []
}
return data as unknown[] as TableMapLog[]
return data as unknown[] as TableAlias[]
}
async getByContext(sourceFileName: string) {
const { data } = await this.db.select(`SELECT * FROM table_map_log
const { data } = await this.db.select(`SELECT * FROM TABLE_ALIASES
WHERE source_file_name=@source_file_name
`, { source_file_name: sourceFileName })
if (!data) {

106
src/datamodel/repository/tableDefinitions.ts Normal file
View file

@ -0,0 +1,106 @@
import { Repository } from "./abstractRepository";
import { v4 as uuidv4 } from "uuid";
export interface TableDefinition {
id: string,
table_name: string,
source_file: string,
created_at: string,
updated_at: string,
file_hash: string,
type: string,
refresh_id: string, // FIXME: we probably don't need this one, it is all solved on the table_name level.
arguments: string[]
}
const parseDbEntry = (entry: TableDefinition): TableDefinition => ({
...entry,
arguments: JSON.parse(entry.arguments ? entry.arguments.toString() : '[]')
})
export type TableDefinitionExternal = Omit<TableDefinition, 'id' | 'created_at' | 'updated_at'>
export class TableDefinitionsRepository extends Repository {
private readonly TABLE_NAME = 'TABLE_DEFINITIONS';
async init() {
await this.createTable()
}
private async createTable() {
await this.db.createTableNoTypes(
this.TABLE_NAME,
['id', 'table_name', 'source_file', 'created_at', 'updated_at', 'file_hash', 'type', 'refresh_id', 'arguments'],
true
)
}
async insert(definition: TableDefinitionExternal) {
const now = Date.now()
await this.db.insertData(this.TABLE_NAME, [{
id: uuidv4(),
table_name: definition.table_name,
source_file: definition.source_file,
file_hash: definition.file_hash,
type: definition.type,
refresh_id: definition.refresh_id,
arguments: JSON.stringify(definition.arguments ?? []),
created_at: now,
updated_at: now
}])
}
async getBySourceFile(sourceFile: string) {
const { data } = await this.db.select(
`SELECT * FROM ${this.TABLE_NAME} WHERE source_file = @sourceFile`,
{ sourceFile }
)
if (!data.length) {
return null
}
return (data as unknown[] as TableDefinition[]).map(parseDbEntry)
}
async getByRefreshId(refreshId: string) {
const { data } = await this.db.select(
`SELECT * FROM ${this.TABLE_NAME} WHERE refresh_id= @refreshId`,
{ refreshId }
)
if (!data.length) {
return null
}
const d = data[0]
return {
...d,
arguments: JSON.parse(d.arguments ? d.arguments.toString() : '[]')
} as TableDefinition
}
async getAll() {
const { data } = await this.db.select(`SELECT * FROM ${this.TABLE_NAME}`, {})
return data.map(d => ({
...d,
type: d.type ?? 'file',
arguments: JSON.parse(d.arguments ? d.arguments.toString() : '[]')
})) as TableDefinition[]
}
async update(id: string, fields: Partial<TableDefinitionExternal>) {
const updateData: Record<string, unknown> = {
...fields,
updated_at: Date.now().toString()
};
// Handle arguments field if it exists in the update
if ('arguments' in fields) {
updateData.arguments = JSON.stringify(fields.arguments);
}
await this.db.db.updateData(this.TABLE_NAME, [{
id,
...updateData
}], 'id')
}
}

142
src/datamodel/sync.ts
View file

@ -1,23 +1,22 @@
import { SqlSealDatabase } from "src/database/database";
import { FileLog, FileLogRepository } from "./repository/fileLog";
import { TableMapLogRepository } from "./repository/tableMapLog";
import { TableAliasesRepository } from "./repository/tableAliases";
import { App, TAbstractFile, TFile, Vault } from "obsidian";
import { FilepathHasher } from "../utils/hasher";
import { Omnibus } from "@hypersphere/omnibus";
import { TableRegistration } from "./types";
import { SyncStrategyFactory } from "./syncStrategy/SyncStrategyFactory";
import { ConfigurationRepository } from "./repository/configuration";
import { TableDefinitionsRepository, TableDefinition } from "./repository/tableDefinitions";
import { ParserTableDefinition } from "./syncStrategy/types";
import { uniq } from "lodash";
const SQLSEAL_DATABASE_VERSION = 1;
const SQLSEAL_DATABASE_VERSION = 2;
export class Sync {
private fileLog: FileLogRepository;
private tableMapLog: TableMapLogRepository;
private tableDefinitionsRepo: TableDefinitionsRepository;
private tableMapLog: TableAliasesRepository;
private bus = new Omnibus()
private tableMaps = new Map<string, FileLog>()
constructor(
private readonly db: SqlSealDatabase,
private readonly vault: Vault,
@ -26,14 +25,6 @@ export class Sync {
}
async updateTableMaps() {
const data = await this.fileLog.getAll()
this.tableMaps.clear()
data.forEach(log => {
this.tableMaps.set(log.file_name, log)
})
}
async init() {
await this.db.connect()
@ -52,21 +43,20 @@ export class Sync {
await config.init()
this.fileLog = new FileLogRepository(this.db)
await this.fileLog.init()
this.tableDefinitionsRepo = new TableDefinitionsRepository(this.db)
await this.tableDefinitionsRepo.init()
this.tableMapLog = new TableMapLogRepository(this.db)
this.tableMapLog = new TableAliasesRepository(this.db)
await this.tableMapLog.init()
await config.setConfig('version', SQLSEAL_DATABASE_VERSION)
const fileLogs = await this.fileLog.getAll()
for (const log of fileLogs) {
await this.syncFileByName(log.file_name)
const fileLogs = await this.tableDefinitionsRepo.getAll()
const uniquePaths = uniq(fileLogs.map(l => l.source_file))
for (const path of uniquePaths) {
await this.syncFileByName(path)
}
await this.updateTableMaps()
// START SYNCING
this.startSync()
}
@ -82,22 +72,28 @@ export class Sync {
async syncFile(file: TFile) {
this.bus.trigger('file::change::'+file.path)
const maps = await this.tableDefinitionsRepo.getBySourceFile(file.path)
const entry = this.tableMaps.get(file.path)
if (!entry) {
if (!maps) {
return
}
if (entry.file_hash !== file.stat.mtime.toString()) {
const syncObject = SyncStrategyFactory.getStrategyByFileLog(entry, this.app)
const { data, columns } = await syncObject.returnData()
const tableName = entry.table_name
await this.db.createTableClean(tableName, columns)
await this.db.insertData(tableName, data)
await this.fileLog.updateHash(tableName, file.stat.mtime.toString())
this.bus.trigger('change::' + tableName)
for (const entry of maps) {
if (!entry) {
return
}
if (entry.file_hash !== file.stat.mtime.toString()) {
const syncObject = SyncStrategyFactory.getStrategy(entry, this.app)
const { data, columns } = await syncObject.returnData()
const tableName = entry.table_name
await this.db.createTableNoTypes(tableName, columns)
await this.db.insertData(tableName, data)
await this.tableDefinitionsRepo.update(entry.id, { file_hash: file.stat.mtime.toString() })
this.bus.trigger('change::' + tableName)
}
}
}
@ -109,9 +105,8 @@ export class Sync {
})
}
async registerFile(log: Omit<FileLog, 'id' | 'created_at' | 'updated_at'>) {
await this.fileLog.insert(log)
await this.updateTableMaps()
async registerFile(log: Omit<TableDefinition, 'id' | 'created_at' | 'updated_at'>) {
await this.tableDefinitionsRepo.insert(log)
}
async getTablesMappingForContext(sourceFileName: string) {
@ -134,43 +129,42 @@ export class Sync {
return `file_${hash}`
}
async registerTable(reg: TableRegistration) {
const existingFileLog = await this.fileLog.getByFilename(reg.fileName)
let fileTableName: string;
const syncObject = SyncStrategyFactory.getStrategy(reg, this.app)
if (!existingFileLog) {
// We need to create new one.
const tableName = await syncObject.tableName()
await this.registerFile({
file_name: reg.fileName,
table_name: tableName,
file_hash: '',
type: reg.type,
extras: reg.extras
})
fileTableName = tableName
// INITIAL SYNC
await this.syncFileByName(reg.fileName)
async registerTable(reg: ParserTableDefinition) {
const syncObject = await SyncStrategyFactory.getStrategyFromParser(reg, this.app)
const definition = syncObject.tableDefinition
const existingDefinition = await this.tableDefinitionsRepo.getByRefreshId(definition.refresh_id) // FIXME: probably can be by table name too.
let tableName: string;
if (!existingDefinition) {
// This one is not registered yet, registering
await this.registerFile(definition)
await this.syncFileByName(definition.source_file)
tableName = definition.table_name
} else {
fileTableName = existingFileLog.table_name
tableName = existingDefinition.table_name
}
const existingTableLog = await this.tableMapLog.getByAlias(reg.sourceFile, reg.aliasName)
// TODO: THIS PART SHOULD BE REWRITTEN SOONish
const existingTableLog = await this.tableMapLog.getByAlias(reg.sourceFile, reg.tableAlias)
if (!existingTableLog) {
// console.log(`Registering new mapping ${reg.sourceFile} :: ${reg.tableAlias} -> ${tableName}`)
// Create new one
await this.tableMapLog.insert({
alias_name: reg.aliasName,
alias_name: reg.tableAlias,
source_file_name: reg.sourceFile,
table_name: fileTableName,
table_name: tableName,
})
} else {
// Already exists, doing nothing.
// Check if it is the same mapping
if (existingTableLog.table_name != fileTableName) {
await this.tableMapLog.deleteMapping(existingTableLog.id) // FIXME: might need to remove event too
if (existingTableLog.table_name !== tableName) {
// console.log(`Alias ${reg.sourceFile} :: ${reg.tableAlias} changed table, now it should refer to ${tableName})`)
await this.tableMapLog.deleteMapping(existingTableLog.id)
await this.tableMapLog.insert({
alias_name: reg.aliasName,
table_name: fileTableName,
alias_name: reg.tableAlias,
table_name: tableName,
source_file_name: reg.sourceFile
})
}
@ -188,22 +182,4 @@ export class Sync {
}
return `change::${log.table_name}`
}
// async onFileChange(file: TFile) {
// const path = file.path
// const hash = file.stat.mtime
// // Checking if the file is registered
// const reg = await this.fileLog.getByFilename(path)
// if (reg) {
// // Need to syncronise it.
// // Need to save update in the database.
// // Need to trigger all related files.
// const allRelated = this.tableMapLog.getByTableName(reg.tableName)
// for (const related of allRelated) {
// this.bus.bus.trigger('')
// }
// }
// // Checking if the file file is in the map.
// }
}

72
src/datamodel/syncStrategy/CsvFileSyncStrategy.ts
View file

@ -1,47 +1,71 @@
import { App } from "obsidian";
import { TableRegistration } from "../types";
import { ISyncStrategy } from "./abstractSyncStrategy";
import { sanitise } from "../../utils/sanitiseColumn";
import { parse } from "papaparse";
import { FieldTypes, toTypeStatements } from "../../utils/typePredictions";
import { FilepathHasher } from "../../utils/hasher";
import { TableDefinitionConfig } from "./types";
import { SourceType } from "../../grammar/newParser";
import { TableDefinitionExternal } from "../repository/tableDefinitions";
import { ParserTableDefinition } from "./types";
import { App } from "obsidian";
export class CsvFileSyncStrategy implements ISyncStrategy {
constructor(private reg: TableRegistration, private app: App) {
const DEFAULT_FILE_HASH = ''
export class CsvFileSyncStrategy extends ISyncStrategy {
static async fromParser(def: ParserTableDefinition, app: App): Promise<CsvFileSyncStrategy> {
const tableSourceFile = def.arguments[0]
const path = app.metadataCache.getFirstLinkpathDest(tableSourceFile, def.sourceFile)
if (!path) {
throw new Error(`File not found: ${tableSourceFile}`)
}
const sourcePath = path.path
const hash = await FilepathHasher.sha256(`${sourcePath}`) // FILENAME is in this case
const tableName = `file_${hash}`
return new CsvFileSyncStrategy({
arguments: def.arguments,
file_hash: DEFAULT_FILE_HASH,
refresh_id: tableName,
source_file: sourcePath,
table_name: tableName,
type: def.type
}, app)
}
async tableName() {
const hash = await FilepathHasher.sha256(`${this.reg.sourceFile}`) // FILENAME is in this case
return `file_${hash}`
}
static processTableDefinition(config: TableDefinitionConfig) {
async toTableDefinition(): Promise<TableDefinitionExternal> {
const tableSourceFile = this.def.arguments[0]
const hash = await FilepathHasher.sha256(`${tableSourceFile}`) // FILENAME is in this case
const def = this.def
const tableName = `file_${hash}`
return {
tableName: config.alias,
type: config.type as SourceType,
fileName: config.arguments
arguments: def.arguments,
file_hash: DEFAULT_FILE_HASH,
refresh_id: tableName,
source_file: tableSourceFile,
table_name: tableName,
type: def.type
}
}
async returnData() {
const file = this.app.vault.getFileByPath(this.reg.sourceFile)!
const file = this.app.vault.getFileByPath(this.def.source_file)!
const data = await this.app.vault.cachedRead(file)
// TODO: PROBABLY SHOULD BE EXTRACTED SOMEWHERE FROM HERE later.
const parsed = parse<Record<string, string>>(data, {
header: true,
dynamicTyping: false,
dynamicTyping: true,
skipEmptyLines: true,
transformHeader: sanitise
})
const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
name: key,
type: value as FieldTypes
}));
// const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
// const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
// name: key,
// type: value as FieldTypes
// }));
return { columns, data: parsed.data }
return { data: parsed.data, columns: parsed.meta.fields ?? [] }
}
}

62
src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts
View file

@ -1,44 +1,32 @@
import { FilepathHasher } from "../../utils/hasher";
import { TableRegistration } from "../types";
import { ISyncStrategy } from "./abstractSyncStrategy";
import type { App } from "obsidian";
import { Component, MarkdownRenderer } from "obsidian";
import { FieldTypes, toTypeStatements } from "../../utils/typePredictions";
import { TableDefinitionConfig } from "./types";
import { SourceType } from "../../grammar/newParser";
import { ParserTableDefinition } from "./types";
const DEFAULT_FILE_HASH = ''
interface TableData {
columns: string[];
data: Array<Record<string, string>>;
}
export class MarkdownTableSyncStrategy extends ISyncStrategy {
export class MarkdownTableSyncStrategy implements ISyncStrategy {
constructor(private reg: TableRegistration, private app: App) {
}
static processTableDefinition(config: TableDefinitionConfig) {
return {
tableName: config.alias,
type: config.type as SourceType,
fileName: config.sourceFile, //+ '?table=' + config.arguments
extras: {
tableNo: parseInt(config.arguments, 10) ?? 0
}
}
}
async tableName() {
const hash = await FilepathHasher.sha256(`${this.reg.sourceFile}__${this.reg.fileName}`) // FILENAME is in this case
return `mdtable_${hash}`
static async fromParser(def: ParserTableDefinition, app: App): Promise<MarkdownTableSyncStrategy> {
const index = def.arguments[0]
const hash = await FilepathHasher.sha256(`${def.sourceFile}`) // FILENAME is in this case
const tableName = `mdtable_${hash}_${index}`
return new MarkdownTableSyncStrategy({
arguments: def.arguments,
file_hash: DEFAULT_FILE_HASH,
refresh_id: tableName,
source_file: def.sourceFile,
table_name: tableName,
type: def.type
}, app)
}
async returnData() {
const tableIndex = this.reg.extras.tableNo
const file = this.app.vault.getFileByPath(this.reg.sourceFile)!
const tableIndex = parseInt(this.def.arguments[0], 10)
const file = this.app.vault.getFileByPath(this.def.source_file)!
const content = await this.app.vault.read(file);
// Create a temporary div to render the markdown
@ -54,13 +42,13 @@ export class MarkdownTableSyncStrategy implements ISyncStrategy {
// Check if the requested table exists
if (tableIndex >= tables.length) {
return null;
return { data: [], columns: [] };
}
const targetTable = tables[tableIndex];
// Extract headers
const headers = Array.from(targetTable.querySelectorAll('th')).map(th => th.textContent?.trim() || '');
const headers = Array.from(targetTable.querySelectorAll('th')).map(th => th.textContent?.trim() || '').filter(x => !!x);
// Extract rows
const rows = Array.from(targetTable.querySelectorAll('tr')).slice(1); // Skip header row
@ -75,15 +63,9 @@ export class MarkdownTableSyncStrategy implements ISyncStrategy {
}, {} as Record<string, string>);
});
const typeStatements = toTypeStatements(headers ?? [], data)
const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
name: key,
type: value as FieldTypes
}));
return {
columns,
data
data,
columns: headers ?? []
};
}
}

61
src/datamodel/syncStrategy/SyncStrategyFactory.ts
View file

@ -1,24 +1,25 @@
import { App } from "obsidian";
import { FileLog } from "../repository/fileLog";
import { TableRegistration } from "../types";
import { ISyncStrategy } from "./abstractSyncStrategy";
import { CsvFileSyncStrategy } from "./CsvFileSyncStrategy";
import { MarkdownTableSyncStrategy } from "./MarkdownTableSyncStrategy";
import { JsonFileSyncStrategy } from "./JSONFileSyncStrategy";
import { ParserTableDefinition } from "./types";
import { TableDefinitionExternal } from "../repository/tableDefinitions";
const fileLogToTableRegistration = (log: FileLog): TableRegistration => {
return {
aliasName: log.table_name,
extras: log.extras,
fileName: log.file_name,
sourceFile: log.file_name, // ???? probably not needed here
type: log.type
}
function getFileExtension(pathname: string): string | null {
// FIXME: splitting for now
return pathname.split('.')[1].toLowerCase()
// const url = new URL(pathname);
// const filePath = url.pathname;
// const fileName = path.basename(filePath);
// return path.extname(fileName);
}
const resolveFileStrategy = (filename: string) => {
const parts = filename.split('.')
const extension = parts[parts.length - 1].toLowerCase()
const extension = getFileExtension(filename)
switch (extension) {
case 'csv':
@ -31,27 +32,25 @@ const resolveFileStrategy = (filename: string) => {
}
}
const resolveClass = (type: string, filename: string) => {
switch (type) {
case 'file':
return resolveFileStrategy(filename)
case 'table':
return MarkdownTableSyncStrategy
default:
throw new Error(`Undefined strategy for type ${type}`)
}
}
export class SyncStrategyFactory {
static getStrategy(reg: TableRegistration, app: App): ISyncStrategy {
switch (reg.type) {
case 'file':
const Cls = resolveFileStrategy(reg.fileName)
return new Cls(reg, app)
case 'table':
return new MarkdownTableSyncStrategy(reg, app)
}
static async getStrategyFromParser(def: ParserTableDefinition, app: App): Promise<ISyncStrategy> {
const Cls = resolveClass(def.type, def.arguments[0])
return Cls.fromParser(def, app)
}
static getStaticStrategyReference(type: string) {
switch (type) {
case 'file':
return CsvFileSyncStrategy
case 'table':
return MarkdownTableSyncStrategy
}
}
static getStrategyByFileLog(log: FileLog, app: App) {
return SyncStrategyFactory.getStrategy(fileLogToTableRegistration(log), app)
static getStrategy(def: TableDefinitionExternal, app: App): ISyncStrategy {
const Cls = resolveClass(def.type, def.source_file)
return new Cls(def, app)
}
}

22
src/datamodel/syncStrategy/abstractSyncStrategy.ts
View file

@ -1,4 +1,20 @@
export interface ISyncStrategy {
tableName(): Promise<string>;
returnData(): any;
import { App } from "obsidian";
import { TableDefinitionExternal } from "../repository/tableDefinitions";
import { ParserTableDefinition } from "./types";
export abstract class ISyncStrategy {
constructor(protected def: TableDefinitionExternal, protected app: App) {
}
get tableDefinition(): TableDefinitionExternal {
return this.def
}
abstract returnData(): Promise<{
data: Record<string, unknown>[],
columns: string[]
}>;
static async fromParser(def: ParserTableDefinition, app: App): Promise<ISyncStrategy> {
throw new Error('Not Defined')
}
}

6
src/datamodel/syncStrategy/types.ts
View file

@ -1,6 +1,6 @@
export interface TableDefinitionConfig {
export interface ParserTableDefinition {
type: string;
alias: string;
arguments: string;
tableAlias: string;
arguments: string[];
sourceFile: string;
}

9
src/datamodel/types.ts
View file

@ -1,9 +0,0 @@
import { SourceType } from "src/grammar/newParser";
export interface TableRegistration {
fileName: string;
aliasName: string;
sourceFile: string;
type: SourceType;
extras: any;
}

1
src/editorExtension/inlineCodeBlock.ts
View file

@ -3,7 +3,6 @@ import { RangeSetBuilder } from "@codemirror/state";
import { syntaxTree } from "@codemirror/language";
import { App } from "obsidian";
import { SqlSealDatabase } from "../database/database";
import { RendererRegistry } from "../renderer/rendererRegistry";
import { Sync } from "../datamodel/sync";
import { SqlSealInlineHandler } from "src/codeblockHandler/inline/InlineCodeHandler";
import { InlineProcessor } from "src/codeblockHandler/inline/InlineProcessor";

78
src/grammar/parser.test.ts → src/grammar/newParser.test.ts
View file

@ -12,11 +12,12 @@ describe('Parser', () => {
it('should parse properly table and select', () => {
expect(parseLanguage(`TABLE x = file(a.csv)
SELECT * FROM files`)).toEqual({
SELECT * FROM files`, "source.md")).toEqual({
tables: [{
tableName: 'x',
fileName: 'a.csv',
type: 'file'
tableAlias: 'x',
arguments: ['a.csv'],
type: 'file',
sourceFile: "source.md"
}],
queryPart: 'SELECT * FROM files',
intermediateContent: ''
@ -26,16 +27,18 @@ SELECT * FROM files`)).toEqual({
it('should parse properly multiple tables in the same file', () => {
expect(parseLanguage(`TABLE x = file(a.csv)
TABLE y = file(very-long-name.csv)
SELECT * FROM a JOIN y ON a.id=y.id`)).toEqual({
SELECT * FROM a JOIN y ON a.id=y.id`, 's.md')).toEqual({
tables: [{
tableName: 'x',
fileName: 'a.csv',
type: 'file'
tableAlias: 'x',
arguments: ['a.csv'],
type: 'file',
sourceFile: 's.md'
},
{
tableName: 'y',
fileName: 'very-long-name.csv',
type: 'file'
tableAlias: 'y',
arguments: ['very-long-name.csv'],
type: 'file',
sourceFile: 's.md'
}],
queryPart: 'SELECT * FROM a JOIN y ON a.id=y.id',
intermediateContent: ''
@ -44,22 +47,24 @@ SELECT * FROM a JOIN y ON a.id=y.id`)).toEqual({
it('should parse properly multiple tables in the same file', () => {
expect(parseLanguage(`TABLE x = file(a.csv)
TABLE y = file(very-long-name.csv)
TABLE y = file( very-long-name.csv )
PLOT {
x: 5,
y: 654
}
SELECT * FROM a JOIN y ON a.id=y.id`)).toEqual({
SELECT * FROM a JOIN y ON a.id=y.id`, 'source.md')).toEqual({
tables: [{
tableName: 'x',
fileName: 'a.csv',
type: 'file'
tableAlias: 'x',
arguments: ['a.csv'],
type: 'file',
sourceFile: 'source.md'
},
{
tableName: 'y',
fileName: 'very-long-name.csv',
type: 'file'
tableAlias: 'y',
arguments: ['very-long-name.csv'],
type: 'file',
sourceFile: 'source.md'
}],
queryPart: 'SELECT * FROM a JOIN y ON a.id=y.id',
intermediateContent: `PLOT {
@ -106,14 +111,15 @@ sElEcT * fRoM files`)).toEqual({
select * from files
`)).toEqual({
`, 'src.md')).toEqual({
'intermediateContent': 'html',
'queryPart': 'select * from files',
tables: [
{
tableName: 'x',
fileName: 'a.csv',
type: 'file'
tableAlias: 'x',
arguments: ['a.csv'],
type: 'file',
sourceFile: 'src.md'
}
]
})
@ -122,14 +128,32 @@ sElEcT * fRoM files`)).toEqual({
it('should properly parse query with just table and sql syntax', () => {
const q = `table x = file(aaa.csv)
select * from x`
expect(parseLanguage(q)).toEqual({
expect(parseLanguage(q, '2025-01-01.md')).toEqual({
queryPart: 'select * from x',
intermediateContent: '',
tables: [
{
tableName: 'x',
fileName: 'aaa.csv',
type: 'file'
tableAlias: 'x',
arguments: ['aaa.csv'],
type: 'file',
sourceFile: '2025-01-01.md'
}
]
})
})
it('should properly parse query with table definition with multiple arguments', () => {
const q = `table x = file(aaa.csv, secondArg, third argument)
select * from x`
expect(parseLanguage(q, '2025-01-01.md')).toEqual({
queryPart: 'select * from x',
intermediateContent: '',
tables: [
{
tableAlias: 'x',
arguments: ['aaa.csv', 'secondArg', 'third argument'],
type: 'file',
sourceFile: '2025-01-01.md'
}
]
})

34
src/grammar/newParser.ts
View file

@ -1,21 +1,7 @@
import { SyncStrategyFactory } from "../datamodel/syncStrategy/SyncStrategyFactory";
import { TableDefinitionConfig } from "../datamodel/syncStrategy/types";
export type SourceType = 'file' | 'table'
export interface Table {
tableName: string;
type: SourceType,
fileName: string;
extras?: Record<string, string | number>
}
export interface TableWithParentPath extends Table {
parentPath: string;
}
import { ParserTableDefinition } from "../datamodel/syncStrategy/types";
interface ParsedLanguage {
tables: Table[];
tables: ParserTableDefinition[];
queryPart: string;
intermediateContent: string;
}
@ -48,21 +34,15 @@ export function parseLanguage(input: string, sourceFile: string = ''): ParsedLan
// Parse TABLE declaration
// Format: TABLE tableName = file(filename.csv)
const tableMatch = line.match(/TABLE\s+(\w+)\s*=\s*(file|table)\(([^)]+)\)/i);
const tableMatch = line.match(/TABLE\s+(\w+)\s*=\s*(\w+)\(([^)]+)\)/i);
if (tableMatch) {
const config = {
alias: tableMatch[1],
tableAlias: tableMatch[1],
type: tableMatch[2].toLowerCase(),
arguments: tableMatch[3],
arguments: tableMatch[3].split(',').map(t => t.trim()),
sourceFile: sourceFile
} satisfies TableDefinitionConfig
const definition = SyncStrategyFactory
.getStaticStrategyReference(config.type)
?.processTableDefinition(config)
if (!definition) {
throw new Error(`Cannot process table for: ${line}`)
}
result.tables.push(definition);
} satisfies ParserTableDefinition
result.tables.push(config);
}
currentPosition++;
}

17
src/utils/typePredictions.test.ts
View file

@ -1,17 +0,0 @@
import { predictJson, predictType } from './typePredictions'
describe('Utils', () => {
it('should properly predict TEXT type', () => {
expect(predictType('xxx', [{ 'xxx': 'dhjkafhkjafhkfas' }])).toEqual('TEXT')
})
it('should properly predict JSON type', () => {
expect(predictType('xxx', [{ 'xxx': { a: 'b' } }])).toEqual('JSON')
})
it('should properly predict json by parsing it', () => {
expect(predictJson([{ 'xxx': '["a", "b"]'}])).toEqual([{
xxx: ['a', 'b']
}])
})
})

99
src/utils/typePredictions.ts
View file

@ -1,99 +0,0 @@
import { parse } from 'json5'
import { camelCase } from "lodash";
export const predictType = (field: string, data: Array<Record<string, string | Object>>) => {
if (field === 'id') {
return 'TEXT'
}
const d = data.map(d => d[field])
// CHECK JSONS.
const isObject = d.reduce((acc, d) => acc && (!d || typeof d === 'object'), true)
if (isObject) {
return 'JSON'
}
const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field] as string), true)
if (canBeNumber) {
// Check if Integer or Float
const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field] as string) === parseInt(d[field] as string), true)
if (canBeInteger) {
return 'INTEGER'
}
return 'REAL'
}
return 'TEXT'
}
export type FieldTypes = ReturnType<typeof predictType>
export const predictJson = (data: Array<Record<string, unknown>>) => {
return data.map(d => Object.keys(d).reduce((o, k) => {
if (typeof d[k] !== 'string') {
return {
...o,
[k]: d[k]
}
}
const v = (d[k] as string).trim()
if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) {
try {
const d = parse(v)
return {
...o,
[k]: d
}
} catch (e) {
}
}
return {
...o,
[k]: d[k]
}
}, {}))
}
export function isNumeric(str: string) {
if (typeof str != "string") return false // we only process strings!
return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
export function dataToCamelCase(data: Array<Record<string, unknown>>) {
return data.map(d => Object.keys(d).reduce((acc, k) => ({
...acc,
[camelCase(k)]: d[k]
}), {}))
}
export const toTypeStatements = (header: Array<string>, data: Array<Record<string, string>>) => {
let d: Array<Record<string, string | number>> = data
const types: Record<string, ReturnType<typeof predictType>> = {}
header.forEach(key => {
const type = predictType(key, data)
if (type === 'REAL' || type === 'INTEGER') {
// converting all data here to text
d = d.map(record => ({
...record,
[key]: type === 'REAL'
? parseFloat(record[key] as string)
: parseInt(record[key] as string)
}))
}
types[key] = type
})
return {
data: d,
types
}
}

2
src/utils/ui.ts
View file

@ -46,7 +46,7 @@ const generateLink = (config: SqlSealAnchorElement, app: App) => {
if (fileHref.endsWith('.md')) {
fileHref = fileHref.slice(0, -3)
}
const vaultName = encodeURIComponent(app.appId);
const vaultName = encodeURIComponent((app as any).appId);
const encodedPath = encodeURIComponent(fileHref);
href = `obsidian://open?vault=${vaultName}&file=${encodedPath}`;
}

27
src/vaultSync/SealFileSync.ts
View file

@ -1,5 +1,4 @@
import { App, Plugin, TFile } from "obsidian";
import { FieldTypes } from "../utils/typePredictions";
import { sanitise } from "../utils/sanitiseColumn";
import { AFileSyncTable } from "./tables/abstractFileSyncTable";
@ -13,7 +12,6 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
}
export class SealFileSync {
private currentSchema: Record<string, FieldTypes> = {}
private tablePlugins: Array<AFileSyncTable> = []
constructor(
public readonly app: App,
@ -23,14 +21,7 @@ export class SealFileSync {
if (!(file instanceof TFile)) {
return
}
await sleep(100)
const frontmatter = await extractFrontmatterFromFile(file, plugin)
if (this.hasNewColumns(frontmatter)) {
await sleep(1000)
await this.init()
return
}
await sleep(100) // FIXME: check if this is actually needed.
await Promise.all(this.tablePlugins.map(p => p.onFileModify(file)))
@ -40,15 +31,6 @@ export class SealFileSync {
if (!(file instanceof TFile)) {
return
}
const frontmatter = await extractFrontmatterFromFile(file, plugin)
if (this.hasNewColumns(frontmatter)) {
await sleep(1000)
await this.init()
return
}
await Promise.all(this.tablePlugins.map(p => p.onFileCreate(file)))
}))
@ -101,11 +83,4 @@ export class SealFileSync {
)
}
hasNewColumns(newFrontmatter: Record<string, any>) {
const currentFields = Object.keys(this.currentSchema)
const newFields = Object.keys(newFrontmatter).filter(f => !currentFields.includes(f))
return newFields.length > 0
}
}

56
src/vaultSync/tables/filesTable.ts
View file

@ -2,10 +2,13 @@ import { App, Plugin, TAbstractFile, TFile } from "obsidian";
import { AFileSyncTable } from "./abstractFileSyncTable";
import { sanitise } from "../../utils/sanitiseColumn";
import { SqlSealDatabase } from "../../database/database";
import { predictType } from "../../utils/typePredictions";
import { difference } from "lodash";
const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
export const FILES_TABLE_NAME = 'files'
const extractFrontmatterFromFile = (file: TFile, plugin: Plugin): Record<string, unknown> => {
const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
return Object.fromEntries(
@ -14,7 +17,7 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
)
}
function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<string, any>) {
function fileData(file: TAbstractFile, { tags: _tags, ...frontmatter }: Record<string, any>) {
return {
...frontmatter,
path: file.path,
@ -24,41 +27,54 @@ function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<st
}
export class FilesFileSyncTable extends AFileSyncTable {
private currentSchema: Record<string, ReturnType<typeof predictType>>
private columns: string[] = []
shouldPerformBulkInsert = true;
constructor(db: SqlSealDatabase, app: App, private readonly plugin: Plugin) {
super(db, app)
}
async onFileModify(file: TFile): Promise<void> {
// we need to update the row
const frontmatter = await extractFrontmatterFromFile(file, this.plugin)
await this.db.updateData('files', [fileData(file, frontmatter)])
const frontmatter = extractFrontmatterFromFile(file, this.plugin)
const frontmatterWithFileData = fileData(file, frontmatter)
const columns = Object.keys(frontmatterWithFileData)
await this.updateColumnsIfNeeded(columns)
await this.db.updateData(FILES_TABLE_NAME, [])
await sleep(1000) // TO DELAY OTHER PLUGINS, UGLY BUT WORKS.
}
async onFileDelete(path: string): Promise<void> {
await this.db.deleteData('files', [{ id: path }])
await this.db.deleteData(FILES_TABLE_NAME, [{ id: path }])
}
async updateColumnsIfNeeded(newSetOfColumns: string[]) {
const newColumns = difference(newSetOfColumns, this.columns)
if (newColumns.length) {
await this.db.addColumns(FILES_TABLE_NAME, newColumns)
this.columns = await this.db.getColumns(FILES_TABLE_NAME)
}
}
async onFileCreate(file: TFile): Promise<void> {
// we need to update the row
const frontmatter = await extractFrontmatterFromFile(file, this.plugin)
// TODO: Check if there are new columns. If there are, we update the database.
await this.db.insertData('files', [fileData(file, frontmatter)])
// FIXME: should we be waiting here?
await sleep(1000)
const frontmatter = extractFrontmatterFromFile(file, this.plugin)
const frontmatterWithFileData = fileData(file, frontmatter)
const columns = Object.keys(frontmatterWithFileData)
await this.updateColumnsIfNeeded(columns)
await this.db.insertData(FILES_TABLE_NAME, [frontmatterWithFileData])
}
async onFileCreateBulk(files: Array<TFile>) {
// FIXME: implement this one and replace the other implementation.
const data = await Promise.all(files.map(async f => fileData(f, await extractFrontmatterFromFile(f, this.plugin))))
const schema = await this.db.createTableWithData('files', data)
this.currentSchema = schema
this.columns = await this.db.getColumns(FILES_TABLE_NAME)
// One by one
for (const file of files) {
await this.onFileCreate(file)
}
}
async onInit(): Promise<void> {
this.db.createTableNoTypes(FILES_TABLE_NAME, ['id', 'name', 'path'], true)
this.columns = await this.db.getColumns(FILES_TABLE_NAME)
}
}

5
src/vaultSync/tables/tagsTable.ts
View file

@ -32,9 +32,6 @@ export class TagsFileSyncTable extends AFileSyncTable {
async onInit(): Promise<void> {
await this.db.createTable('tags', {
'tag': 'TEXT',
'fileId': 'TEXT'
})
await this.db.createTableNoTypes('tags', ['tag', 'fileId'])
}
}

6
src/vaultSync/tables/tasksTable.ts
View file

@ -45,10 +45,6 @@ export class TasksFileSyncTable extends AFileSyncTable {
}
async onInit(): Promise<void> {
await this.db.createTable('tasks', {
'task': 'TEXT',
'completed': 'INTEGER',
'filePath': 'TEXT'
})
await this.db.createTableNoTypes('tasks', ['task', 'completed', 'filePath'])
}
}

3
versions.json
View file

@ -25,5 +25,6 @@
"0.15.0": "0.15.0",
"0.16.0": "0.15.0",
"0.16.1": "0.15.0",
"0.17.0": "0.15.0"
"0.17.0": "0.15.0",
"0.18.0": "0.15.0"
}