mirror of
https://github.com/h-sphere/sql-seal.git
synced 2026-07-22 10:10:28 +00:00
Merge pull request #28 from h-sphere/feat/reusing-database-data
Reworking plugin core code
This commit is contained in:
Kacper Kula
GitHub
commit
d0a76c1daa
47 changed files with 792 additions and 1212 deletions
|
|
@ -1,3 +1,9 @@
|
|||
# 0.13.0
|
||||
Huge upgrade to the code codebase. SQLSeal should be now much faster and more reliable thanks to the following:
|
||||
- Rewritten how files are synched - now each CSV file creates AT MOST one table in the database (synchronisations are being reused accross files)
|
||||
- Rewritten SQL parser - this enables using more advanced SQLite functionality like recursive CTEs, UNIONS and `json_each`!
|
||||
- (minor, technical): code got restructured and fixed so it's easier to contribute.
|
||||
|
||||
# 0.12.4
|
||||
- added "Create CSV file" option in context menu in file explorer.
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ const workerPlugin = {
|
|||
build.onLoad({ filter: /.*/, namespace: 'worker-code' }, async () => {
|
||||
// Build worker code
|
||||
const result = await esbuild.build({
|
||||
entryPoints: ['src/database-worker.ts'],
|
||||
entryPoints: ['src/database/worker/database.ts'],
|
||||
bundle: true,
|
||||
write: false,
|
||||
format: 'iife',
|
||||
|
|
@ -83,7 +83,7 @@ const context = await esbuild.context({
|
|||
banner: {
|
||||
js: banner,
|
||||
},
|
||||
entryPoints: ["main.ts"],
|
||||
entryPoints: ["src/main.ts"],
|
||||
bundle: true,
|
||||
external: [
|
||||
"obsidian",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"id": "sqlseal",
|
||||
"name": "SQLSeal",
|
||||
"version": "0.12.4",
|
||||
"version": "0.13.0",
|
||||
"minAppVersion": "0.15.0",
|
||||
"description": "Use SQL in your notes to query your vault files and CSV content.",
|
||||
"author": "hypersphere",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "sqlseal",
|
||||
"version": "0.12.4",
|
||||
"version": "0.13.0",
|
||||
"description": "A plugin for Obsidian that allows you to run SQL queries on your notes.",
|
||||
"main": "main.js",
|
||||
"scripts": {
|
||||
|
|
@ -42,6 +42,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@ag-grid-community/theming": "^32.3.3",
|
||||
"@hypersphere/omnibus": "^0.1.6",
|
||||
"@jlongster/sql.js": "^1.6.7",
|
||||
"absurd-sql": "^0.0.54",
|
||||
"ag-grid-community": "^32.3.3",
|
||||
|
|
@ -52,6 +53,7 @@
|
|||
"markdown-table-ts": "^1.0.3",
|
||||
"node-sql-parser": "^5.3.4",
|
||||
"papaparse": "^5.4.1",
|
||||
"util": "^0.12.5"
|
||||
"util": "^0.12.5",
|
||||
"uuid": "^11.0.4"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ importers:
|
|||
'@ag-grid-community/theming':
|
||||
specifier: ^32.3.3
|
||||
version: 32.3.3
|
||||
'@hypersphere/omnibus':
|
||||
specifier: ^0.1.6
|
||||
version: 0.1.6
|
||||
'@jlongster/sql.js':
|
||||
specifier: ^1.6.7
|
||||
version: 1.6.7
|
||||
|
|
@ -44,6 +47,9 @@ importers:
|
|||
util:
|
||||
specifier: ^0.12.5
|
||||
version: 0.12.5
|
||||
uuid:
|
||||
specifier: ^11.0.4
|
||||
version: 11.0.4
|
||||
devDependencies:
|
||||
'@jest/globals':
|
||||
specifier: ^29.7.0
|
||||
|
|
@ -719,6 +725,9 @@ packages:
|
|||
resolution: {integrity: sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==}
|
||||
engines: {node: '>=18.18'}
|
||||
|
||||
'@hypersphere/omnibus@0.1.6':
|
||||
resolution: {integrity: sha512-agZuKyhdW0n1JoLYZUuA6Du1QoQn39/LapFgRtbJs7fyRM62C9O2PWISHUCwAKnC1Splshpd8glQgx5pA2zkCg==}
|
||||
|
||||
'@iconify-json/simple-icons@1.2.12':
|
||||
resolution: {integrity: sha512-lRNORrIdeLStShxAjN6FgXE1iMkaAgiAHZdP0P0GZecX91FVYW58uZnRSlXLlSx5cxMoELulkAAixybPA2g52g==}
|
||||
|
||||
|
|
@ -2974,6 +2983,10 @@ packages:
|
|||
util@0.12.5:
|
||||
resolution: {integrity: sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==}
|
||||
|
||||
uuid@11.0.4:
|
||||
resolution: {integrity: sha512-IzL6VtTTYcAhA/oghbFJ1Dkmqev+FpQWnCBaKq/gUluLxliWvO8DPFWfIviRmYbtaavtSQe4WBL++rFjdcGWEg==}
|
||||
hasBin: true
|
||||
|
||||
v8-to-istanbul@9.3.0:
|
||||
resolution: {integrity: sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==}
|
||||
engines: {node: '>=10.12.0'}
|
||||
|
|
@ -3643,6 +3656,8 @@ snapshots:
|
|||
|
||||
'@humanwhocodes/retry@0.4.1': {}
|
||||
|
||||
'@hypersphere/omnibus@0.1.6': {}
|
||||
|
||||
'@iconify-json/simple-icons@1.2.12':
|
||||
dependencies:
|
||||
'@iconify/types': 2.0.0
|
||||
|
|
@ -6313,6 +6328,8 @@ snapshots:
|
|||
is-typed-array: 1.1.13
|
||||
which-typed-array: 1.1.15
|
||||
|
||||
uuid@11.0.4: {}
|
||||
|
||||
v8-to-istanbul@9.3.0:
|
||||
dependencies:
|
||||
'@jridgewell/trace-mapping': 0.3.25
|
||||
|
|
|
|||
|
|
@ -1,101 +0,0 @@
|
|||
import { App, MarkdownPostProcessorContext } from "obsidian"
|
||||
import { displayError, displayNotice } from "./ui"
|
||||
import { resolveFrontmatter } from "./frontmatter"
|
||||
import { hashString } from "./hash"
|
||||
import { extractCtes, prefixedIfNotGlobal, updateTables } from "./sqlReparseTables"
|
||||
import { SqlSealDatabase } from "./database"
|
||||
import { Logger } from "./logger"
|
||||
import { TablesManager } from "./dataLoader/collections/tablesManager"
|
||||
import { QueryManager } from "./dataLoader/collections/queryManager"
|
||||
import { parseLanguage, Table, TableWithParentPath } from "./grammar/newParser"
|
||||
import { RendererRegistry, RenderReturn } from "./rendererRegistry"
|
||||
|
||||
export class SqlSealCodeblockHandler {
|
||||
get globalTables() {
|
||||
return ['files', 'tags', 'tasks', 'xyz'] // Make this come from SealFileSync and plugins.
|
||||
}
|
||||
constructor(
|
||||
private readonly app: App,
|
||||
private readonly db: SqlSealDatabase,
|
||||
private logger: Logger,
|
||||
private tableManager: TablesManager,
|
||||
private queryManager: QueryManager,
|
||||
private rendererRegistry: RendererRegistry
|
||||
) {
|
||||
}
|
||||
|
||||
setupTableSignals(tables: Array<TableWithParentPath>) {
|
||||
tables.forEach(t => {
|
||||
this.tableManager.registerTable(t.tableName, t.fileName, t.parentPath)
|
||||
})
|
||||
}
|
||||
|
||||
setupQuerySignals({ statement, tables }: ReturnType<typeof updateTables>, renderer: RenderReturn, ctx: MarkdownPostProcessorContext, el: Element) {
|
||||
const frontmatter = resolveFrontmatter(ctx, this.app)
|
||||
const renderSelect = async () => {
|
||||
try {
|
||||
const { data, columns } = await this.db.select(statement, frontmatter ?? {})
|
||||
renderer.render({ data, columns })
|
||||
} catch (e) {
|
||||
renderer.error(e.toString())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const sig = this.queryManager.registerQuery(ctx.docId, tables)
|
||||
const unsubscribe = sig(() => {
|
||||
if (!el.isConnected) {
|
||||
unsubscribe()
|
||||
return
|
||||
}
|
||||
renderSelect()
|
||||
})
|
||||
}
|
||||
|
||||
getHandler() {
|
||||
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
|
||||
const prefix = hashString(ctx.sourcePath)
|
||||
|
||||
let results;
|
||||
let renderer: RenderReturn;
|
||||
|
||||
try {
|
||||
await this.db.connect()
|
||||
// Before we display data, we need to parse query to see if there is query part.
|
||||
results = parseLanguage(source)
|
||||
if (results.queryPart) {
|
||||
// FIXME: this one probably needs both renderer and error api.
|
||||
renderer = this.rendererRegistry.prepareRender(results.intermediateContent)(el)
|
||||
} else {
|
||||
displayNotice(el, `Creating tables: ${results.tables.map(t => t.tableName).join(', ')}`)
|
||||
}
|
||||
|
||||
const prefixedTables = results.tables.map(t => {
|
||||
return {
|
||||
...t,
|
||||
tableName: prefixedIfNotGlobal(t.tableName, this.globalTables, prefix),
|
||||
parentPath: ctx.sourcePath
|
||||
}
|
||||
}) satisfies TableWithParentPath[]
|
||||
|
||||
this.setupTableSignals(prefixedTables)
|
||||
|
||||
} catch (e) {
|
||||
displayError(el, e.toString())
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (results.queryPart) {
|
||||
const { statement, tables } = updateTables(results.queryPart!, [...this.globalTables], prefix)
|
||||
const ctes = extractCtes(results.queryPart)
|
||||
const tablesWithoutCtes = tables.filter(t => !ctes.includes(t))
|
||||
this.setupQuerySignals({ statement, tables: tablesWithoutCtes }, renderer!, ctx, el)
|
||||
}
|
||||
} catch (e) {
|
||||
renderer!.error(e.toString())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
97
src/codeblockHandler/CodeblockProcessor.ts
Normal file
97
src/codeblockHandler/CodeblockProcessor.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { OmnibusRegistrator } from "@hypersphere/omnibus";
|
||||
import { App, MarkdownPostProcessorContext, MarkdownRenderChild } from "obsidian";
|
||||
import { SqlSealDatabase } from "src/database/database";
|
||||
import { Sync } from "src/datamodel/sync";
|
||||
import { transformQuery } from "src/sql/transformer";
|
||||
import { parseLanguage, Table } from "src/grammar/newParser";
|
||||
import { RendererRegistry, RenderReturn } from "src/renderer/rendererRegistry";
|
||||
import { displayError, displayNotice } from "src/utils/ui";
|
||||
|
||||
export class CodeblockProcessor extends MarkdownRenderChild {
|
||||
|
||||
registrator: OmnibusRegistrator
|
||||
renderer: RenderReturn
|
||||
|
||||
constructor(
|
||||
private el: HTMLElement,
|
||||
private source: string,
|
||||
private ctx: MarkdownPostProcessorContext,
|
||||
private rendererRegistry: RendererRegistry,
|
||||
private db: SqlSealDatabase,
|
||||
private app: App,
|
||||
private sync: Sync) {
|
||||
super(el)
|
||||
this.registrator = this.sync.getRegistrator()
|
||||
}
|
||||
|
||||
query: string;
|
||||
|
||||
async onload() {
|
||||
try {
|
||||
const results = parseLanguage(this.source)
|
||||
if (results.tables) {
|
||||
await this.registerTables(results.tables)
|
||||
if (!results.queryPart) {
|
||||
displayNotice(this.el, `Creating tables: ${results.tables.map(t => t.tableName).join(', ')}`)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
this.renderer = this.rendererRegistry.prepareRender(results.intermediateContent)(this.el)
|
||||
|
||||
// FIXME: probably should save the one before transform and perform transform every time we execute it.
|
||||
this.query = results.queryPart
|
||||
await this.render()
|
||||
} catch (e) {
|
||||
displayError(this.el, e.toString())
|
||||
}
|
||||
}
|
||||
|
||||
onunload() {
|
||||
this.registrator.offAll()
|
||||
}
|
||||
|
||||
async render() {
|
||||
try {
|
||||
|
||||
const registeredTablesForContext = await this.sync.getTablesMappingForContext(this.ctx.sourcePath)
|
||||
const tranformedQuery = transformQuery(this.query, registeredTablesForContext)
|
||||
|
||||
this.registrator.offAll()
|
||||
Object.values(registeredTablesForContext).forEach(v => {
|
||||
this.registrator.on(`change::${v}`, () => {
|
||||
this.render()
|
||||
})
|
||||
this.registrator.on('file::change::'+this.ctx.sourcePath, () => {
|
||||
sleep(250).then(() => this.render())
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
const file = this.app.vault.getFileByPath(this.ctx.sourcePath)
|
||||
if (!file) {
|
||||
return
|
||||
}
|
||||
const fileCache = this.app.metadataCache.getFileCache(file)
|
||||
const { data, columns } = await this.db.select(tranformedQuery, fileCache?.frontmatter ?? {})
|
||||
this.renderer.render({ data, columns })
|
||||
} catch (e) {
|
||||
this.renderer.error(e.toString())
|
||||
}
|
||||
}
|
||||
|
||||
async registerTables(tables: Table[]) {
|
||||
await Promise.all(tables.map((table) => {
|
||||
const path = this.app.metadataCache.getFirstLinkpathDest(table.fileName, this.ctx.sourcePath)
|
||||
if (!path) {
|
||||
throw new Error(`File does not exist: ${table.fileName} (for ${table.tableName}).`)
|
||||
}
|
||||
return this.sync.registerTable({
|
||||
aliasName: table.tableName,
|
||||
fileName: path.path,
|
||||
sourceFile: this.ctx.sourcePath
|
||||
})
|
||||
}))
|
||||
}
|
||||
}
|
||||
31
src/codeblockHandler/SqlSealCodeblockHandler.ts
Normal file
31
src/codeblockHandler/SqlSealCodeblockHandler.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import { App, MarkdownPostProcessorContext } from "obsidian"
|
||||
import { SqlSealDatabase } from "../database/database"
|
||||
import { RendererRegistry } from "../renderer/rendererRegistry"
|
||||
import { Sync } from "../datamodel/sync"
|
||||
import { CodeblockProcessor } from "./CodeblockProcessor"
|
||||
|
||||
|
||||
export class SqlSealCodeblockHandler {
|
||||
constructor(
|
||||
private readonly app: App,
|
||||
private readonly db: SqlSealDatabase,
|
||||
private sync: Sync,
|
||||
private rendererRegistry: RendererRegistry
|
||||
) {
|
||||
}
|
||||
|
||||
getHandler() {
|
||||
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
|
||||
const processor = new CodeblockProcessor(
|
||||
el,
|
||||
source,
|
||||
ctx,
|
||||
this.rendererRegistry,
|
||||
this.db,
|
||||
this.app,
|
||||
this.sync
|
||||
)
|
||||
ctx.addChild(processor)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
import { Signal, SignalUnsubscriber } from "src/utils/signal";
|
||||
import { dataTransformer, DataTransformerOut } from "../dataTransformer";
|
||||
import { CSVData, csvFileSignal } from "../csvFile";
|
||||
import { App, TFile, Vault } from "obsidian";
|
||||
import { isNull } from "lodash";
|
||||
import { SqlSealDatabase } from "src/database";
|
||||
import { SyncModel } from "src/models/sync";
|
||||
|
||||
export class FilesManager {
|
||||
files: Map<string, Signal<DataTransformerOut>> = new Map()
|
||||
inputFiles: Map<string, Signal<CSVData>> = new Map()
|
||||
unregisters: Array<SignalUnsubscriber> = []
|
||||
syncModel: SyncModel
|
||||
constructor(private vault: Vault, private app: App, private db: SqlSealDatabase) {
|
||||
this.syncModel = new SyncModel(this.db)
|
||||
this.vault.on('modify', async (file) => {
|
||||
if (this.files.has(file.path)) {
|
||||
this.inputFiles.get(file.path)!(await this.loadFile(file.path))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private async loadFile(url: string, sourcePath?: string) {
|
||||
let file: TFile | null;
|
||||
if (sourcePath) {
|
||||
file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
} else {
|
||||
file = this.vault.getFileByPath(url)
|
||||
}
|
||||
if (!file) {
|
||||
return ''
|
||||
}
|
||||
const data = await this.vault.cachedRead(file)
|
||||
return data
|
||||
}
|
||||
|
||||
getFile(url: string, sourcePath: string = '/') {
|
||||
return this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
}
|
||||
|
||||
getFileHandler(url: string, sourcePath?: string) {
|
||||
let file: TFile | null;
|
||||
if (sourcePath) {
|
||||
file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
} else {
|
||||
file = this.vault.getFileByPath(url)
|
||||
}
|
||||
}
|
||||
|
||||
doesFileExist(url: string, sourcePath: string) {
|
||||
const file = this.app.metadataCache.getFirstLinkpathDest(url, sourcePath)
|
||||
return !isNull(file)
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.unregisters.forEach(u => u())
|
||||
}
|
||||
|
||||
private saveManagedRecord(filename: string, checksum: string) {
|
||||
|
||||
}
|
||||
|
||||
getFileSignal(filename: string): Signal<DataTransformerOut> {
|
||||
if (this.files.has(filename)) {
|
||||
return this.files.get(filename)!
|
||||
}
|
||||
const fileSignal = csvFileSignal(filename)
|
||||
const signal = dataTransformer(fileSignal)
|
||||
|
||||
// Check if it's already in the database, if so, we can just load it again.
|
||||
this.db
|
||||
|
||||
// FIXME: add here ability to watch changes.
|
||||
|
||||
this.files.set(filename, signal)
|
||||
this.inputFiles.set(filename, fileSignal)
|
||||
this.loadFile(filename).then(fileSignal)
|
||||
|
||||
return signal
|
||||
}
|
||||
|
||||
addUnregister(unreg: SignalUnsubscriber) {
|
||||
this.unregisters.push(unreg)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
import { createSignal, derivedSignal, SignalUnsubscriber, withSignals } from "src/utils/signal";
|
||||
import { TablesManager } from "./tablesManager";
|
||||
|
||||
export class QueryManager {
|
||||
private registeredQueries: Map<string, SignalUnsubscriber> = new Map()
|
||||
constructor(private tablesManager: TablesManager) { }
|
||||
|
||||
registerQuery(fileId: string, tables: Array<string>) {
|
||||
if (this.registeredQueries.has(fileId)) {
|
||||
this.registeredQueries.get(fileId)!()
|
||||
this.registeredQueries.delete(fileId)
|
||||
}
|
||||
const tableSignals = tables.map(t => this.tablesManager.getTableSignal(t, true))
|
||||
const quertySignal = createSignal<number>()
|
||||
const unregister = withSignals(...tableSignals)(() => {
|
||||
quertySignal(Date.now())
|
||||
})
|
||||
this.registeredQueries.set(fileId, unregister)
|
||||
return quertySignal
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
import { SqlSealDatabase } from "src/database";
|
||||
import { FilesManager } from "./filesManager";
|
||||
import { createSignal, Signal, SignalUnsubscriber } from "src/utils/signal";
|
||||
import { linkTableWithFile } from "../tableSignal";
|
||||
import { FileManager } from "obsidian";
|
||||
import { isNull } from "lodash";
|
||||
|
||||
interface Definition {
|
||||
fileName: string;
|
||||
unlink: SignalUnsubscriber;
|
||||
}
|
||||
|
||||
|
||||
export class TablesManager {
|
||||
private tableLinks: Map<string, Definition> = new Map()
|
||||
private tableSignals: Map<string, Signal<number>> = new Map()
|
||||
constructor(private filesManager: FilesManager, private db: SqlSealDatabase) {
|
||||
|
||||
}
|
||||
|
||||
registerTable(tableName: string, fileName: string, parentPath: string) {
|
||||
const file = this.filesManager.getFile(fileName, parentPath)
|
||||
if (isNull(file)) {
|
||||
throw new Error(`File ${fileName} does not exist`)
|
||||
}
|
||||
|
||||
if (this.tableLinks.has(tableName)) {
|
||||
const { fileName: prevFileName, unlink } = this.tableLinks.get(tableName)!
|
||||
if (prevFileName === file.path) {
|
||||
return
|
||||
}
|
||||
unlink()
|
||||
this.tableLinks.delete(tableName)
|
||||
}
|
||||
const resolvedFileName = file.path
|
||||
const fileSignal = this.filesManager.getFileSignal(resolvedFileName)
|
||||
const tableSignal = this.getTableSignal(tableName)
|
||||
|
||||
const unlink = linkTableWithFile(fileSignal, tableSignal, tableName, this.db)
|
||||
this.tableLinks.set(tableName, {
|
||||
fileName: resolvedFileName,
|
||||
unlink
|
||||
})
|
||||
}
|
||||
|
||||
getTableSignal(tableName: string, failOnUndefined: boolean = false) {
|
||||
if (!this.tableSignals.has(tableName)) {
|
||||
if (failOnUndefined) {
|
||||
throw new Error(`${tableName} does not exist`)
|
||||
}
|
||||
this.tableSignals.set(tableName, createSignal<number>())
|
||||
}
|
||||
return this.tableSignals.get(tableName)!
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
import { TFile } from "obsidian";
|
||||
import { createSignal, Signal } from "src/utils/signal";
|
||||
|
||||
export type CSVData = string
|
||||
|
||||
export const csvFileSignal = (_file: string): Signal<CSVData> => {
|
||||
const sig = createSignal<CSVData>()
|
||||
return sig
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
import { describe, it, expect, jest } from '@jest/globals'
|
||||
import { createSignal } from '../utils/signal'
|
||||
import { dataTransformer } from './dataTransformer'
|
||||
|
||||
describe('Data Transformer', () => {
|
||||
it('should properly transform basic file', () => {
|
||||
const s = createSignal<string>()
|
||||
const dt = dataTransformer(s)
|
||||
s(`a,b,c
|
||||
1,2,3
|
||||
4,5,6`)
|
||||
expect(dt.value.data).toHaveLength(2)
|
||||
expect(dt.value.data[0]).toEqual({
|
||||
a: 1,
|
||||
b: 2,
|
||||
c: 3
|
||||
})
|
||||
|
||||
expect(dt.value.types).toEqual({
|
||||
a: 'INTEGER',
|
||||
b: 'INTEGER',
|
||||
c: 'INTEGER'
|
||||
})
|
||||
})
|
||||
|
||||
it('should properly transform incorrect keys', () => {
|
||||
const s = createSignal<string>()
|
||||
const dt = dataTransformer(s)
|
||||
s(`BEGIN,PLAN,QUERY,RAISE,beak size (mm)
|
||||
1.5,hello world,343.423,22.34,2`)
|
||||
expect(dt.value.types).toEqual({
|
||||
'begin_': 'REAL',
|
||||
'plan_': 'TEXT',
|
||||
'query_': 'REAL',
|
||||
'raise_': 'REAL',
|
||||
'beak_size__mm_': 'INTEGER'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
import { derivedSignal, Signal, SignalEventType } from "../utils/signal"
|
||||
import { CSVData } from "./csvFile"
|
||||
import { parse } from 'papaparse'
|
||||
import { toTypeStatements } from "../utils"
|
||||
import { sanitise } from "../utils/sanitiseColumn"
|
||||
import { TFile } from "obsidian"
|
||||
|
||||
|
||||
export const parseData = (csvData: string) => {
|
||||
const parsed = parse<Record<string, string>>(csvData, {
|
||||
header: true,
|
||||
dynamicTyping: false,
|
||||
skipEmptyLines: true,
|
||||
transformHeader: sanitise
|
||||
})
|
||||
return parsed
|
||||
}
|
||||
|
||||
|
||||
export const dataTransformer = (s: Signal<CSVData>) => {
|
||||
const sig = derivedSignal([s], (data) => {
|
||||
try {
|
||||
const parsed = parseData(data)
|
||||
const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
|
||||
return typeStatements
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return {
|
||||
data: [],
|
||||
types: {}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return sig
|
||||
}
|
||||
|
||||
export type DataTransformerOut = SignalEventType<ReturnType<typeof dataTransformer>>
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
import { Signal } from "src/utils/signal";
|
||||
import { DataTransformerOut } from "./dataTransformer";
|
||||
import { SqlSealDatabase } from "src/database";
|
||||
import { FieldTypes } from "src/utils";
|
||||
|
||||
export const linkTableWithFile = (dataSig: Signal<DataTransformerOut>, tableSignal: Signal<number>, tableName: string, db: SqlSealDatabase) => {
|
||||
return dataSig(({ data, types }) => {
|
||||
|
||||
// Check if the columns are exactly the same. If not, delete and reinstantiate the table.
|
||||
|
||||
// For now always remove and reinstantiate the table
|
||||
db.dropTable(tableName)
|
||||
|
||||
// compute the types of the keys
|
||||
const columns = Object.entries(types).map(([key, value]) => ({
|
||||
name: key,
|
||||
type: value as FieldTypes
|
||||
}));
|
||||
|
||||
// create table again
|
||||
db.createTableClean(tableName, columns)
|
||||
|
||||
// Load all the data
|
||||
db.insertData(tableName, data)
|
||||
|
||||
// Here we do the actual update of the table. If it succeeds, we return it with current date to indicate sync time.
|
||||
tableSignal(Date.now())
|
||||
})
|
||||
}
|
||||
|
|
@ -1,12 +1,8 @@
|
|||
import { App } from "obsidian"
|
||||
import { FieldTypes, toTypeStatements } from "./utils"
|
||||
import { sanitise } from "./utils/sanitiseColumn"
|
||||
import initSqlJs, { BindParams, Database, Statement } from 'sql.js'
|
||||
import wasmBinary from '../node_modules/sql.js/dist/sql-wasm.wasm'
|
||||
import { FieldTypes, toTypeStatements } from "../utils/typePredictions"
|
||||
import * as Comlink from 'comlink'
|
||||
import workerCode from 'virtual:worker-code'
|
||||
import { WorkerDatabase } from "./database-worker"
|
||||
|
||||
import { WorkerDatabase } from "./worker/database";
|
||||
|
||||
export interface FieldDefinition {
|
||||
name: string;
|
||||
|
|
@ -96,21 +92,6 @@ export class SqlSealDatabase {
|
|||
return schema
|
||||
}
|
||||
|
||||
// async addNewColumns(name: string, data: Array<Record<string, unknown>>) {
|
||||
// const schema = await this.getSchema(data)
|
||||
// const currentSchema = this.toObjectsArray(this.db.prepare(`PRAGMA table_info(${name})`))
|
||||
// const currentFields = currentSchema.map((f: any) => f.name)
|
||||
// const newFields = Object.keys(schema).filter(f => !currentFields.includes(f))
|
||||
|
||||
// if (newFields.length === 0) {
|
||||
// return
|
||||
// }
|
||||
|
||||
// const alter = this.db.prepare(`ALTER TABLE ${name} ADD
|
||||
// COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`)
|
||||
// alter.run()
|
||||
// }
|
||||
|
||||
async updateData(name: string, data: Array<Record<string, unknown>>) {
|
||||
return this.db.updateData(name, data)
|
||||
}
|
||||
|
|
@ -135,8 +116,8 @@ export class SqlSealDatabase {
|
|||
await this.createTable(name, fieldsToRecord)
|
||||
}
|
||||
|
||||
async createTable(name: string, fields: Record<string, FieldTypes>) {
|
||||
await this.db.createTable(name, fields)
|
||||
async createTable(name: string, fields: Record<string, FieldTypes>, noDrop?: boolean) {
|
||||
await this.db.createTable(name, fields, noDrop)
|
||||
}
|
||||
async getSchema(data: Array<Record<string, unknown>>) {
|
||||
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
import * as Comlink from "comlink"
|
||||
import initSqlJs from '@jlongster/sql.js';
|
||||
import wasmBinary from '../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm'
|
||||
import wasmBinary from '../../../node_modules/@jlongster/sql.js/dist/sql-wasm.wasm'
|
||||
import { SQLiteFS } from 'absurd-sql';
|
||||
import IndexedDBBackend from '../node_modules/absurd-sql/dist/indexeddb-backend.js';
|
||||
import IndexedDBBackend from '../../../node_modules/absurd-sql/dist/indexeddb-backend.js';
|
||||
import type { BindParams, Database, Statement } from "sql.js";
|
||||
import { sanitise } from "./utils/sanitiseColumn";
|
||||
import { FieldTypes } from "./utils";
|
||||
import { sanitise } from "../../utils/sanitiseColumn";
|
||||
import { FieldTypes } from "../../utils/typePredictions";
|
||||
|
||||
|
||||
function toObjectArray(stmt: Statement) {
|
||||
|
|
@ -112,20 +112,23 @@ export class WorkerDatabase {
|
|||
})
|
||||
}
|
||||
|
||||
async createTable(tableName: string, fields: Record<string, FieldTypes>) {
|
||||
async createTable(tableName: string, fields: Record<string, FieldTypes>, noDrop: boolean = false) {
|
||||
const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
|
||||
const uniqueFields = [...new Map(transformedFiels.map(item =>
|
||||
[item[0], item])).values()]
|
||||
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)
|
||||
// FIXME: probably use schema generator, for now create with hardcoded fields
|
||||
await this.dropTable(tableName)
|
||||
if (!noDrop) {
|
||||
await this.dropTable(tableName)
|
||||
}
|
||||
const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} (
|
||||
${sqlFields.join(', ')}
|
||||
);`
|
||||
|
||||
this.db.prepare(createSQL).run()
|
||||
await this.clearTable(tableName)
|
||||
|
||||
if (!noDrop) {
|
||||
await this.clearTable(tableName)
|
||||
}
|
||||
// Dropping data.
|
||||
}
|
||||
|
||||
|
|
@ -154,10 +157,10 @@ export class WorkerDatabase {
|
|||
}
|
||||
}
|
||||
|
||||
async updateData(tableName: string, data: Array<Record<string, unknown>>) {
|
||||
async updateData(tableName: string, data: Array<Record<string, unknown>>, matchKey: string = 'id') {
|
||||
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
|
||||
data.forEach((d: Record<string, unknown>) => {
|
||||
const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`)
|
||||
const stmt = this.db.prepare(`UPDATE ${tableName} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE ${matchKey} = @${matchKey}`)
|
||||
stmt.run(recordToBindParams(d))
|
||||
})
|
||||
}
|
||||
5
src/datamodel/repository/abstractRepository.ts
Normal file
5
src/datamodel/repository/abstractRepository.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import { SqlSealDatabase } from "src/database/database";
|
||||
|
||||
export abstract class Repository {
|
||||
constructor(protected readonly db: SqlSealDatabase) { }
|
||||
}
|
||||
65
src/datamodel/repository/fileLog.ts
Normal file
65
src/datamodel/repository/fileLog.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import { Repository } from "./abstractRepository";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
|
||||
export interface FileLog {
|
||||
id: string,
|
||||
table_name: string,
|
||||
file_name: string,
|
||||
created_at: string,
|
||||
updated_at: string,
|
||||
file_hash: string
|
||||
}
|
||||
|
||||
export class FileLogRepository extends Repository {
|
||||
|
||||
async init() {
|
||||
await this.createTable()
|
||||
}
|
||||
|
||||
private async createTable() {
|
||||
await this.db.createTable('file_log', {
|
||||
'id': 'TEXT',
|
||||
'table_name': 'TEXT',
|
||||
'file_name': 'TEXT',
|
||||
'created_at': 'TEXT',
|
||||
'updated_at': 'TEXT',
|
||||
'file_hash': 'TEXT'
|
||||
}, true)
|
||||
}
|
||||
|
||||
async insert(log: Omit<FileLog, 'id' | 'created_at' | 'updated_at'>) {
|
||||
const now = Date.now()
|
||||
await this.db.insertData('file_log', [{
|
||||
id: uuidv4(),
|
||||
table_name: log.table_name,
|
||||
file_name: log.file_name,
|
||||
file_hash: log.file_hash,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
}])
|
||||
}
|
||||
|
||||
async getByFilename(filename: string) {
|
||||
const { data } = await this.db.select('SELECT * FROM file_log WHERE file_name = @filename', {
|
||||
filename: filename
|
||||
})
|
||||
|
||||
if (!data.length) {
|
||||
return null
|
||||
}
|
||||
return data[0] as unknown as FileLog
|
||||
}
|
||||
|
||||
async getAll() {
|
||||
const { data } = await this.db.select('SELECT * FROM file_log', {})
|
||||
return data as unknown[] as FileLog[];
|
||||
}
|
||||
|
||||
async updateHash(tableName: string, newHash: string) {
|
||||
await this.db.db.updateData('file_log', [{
|
||||
table_name: tableName,
|
||||
file_hash: newHash
|
||||
}], 'table_name')
|
||||
|
||||
}
|
||||
}
|
||||
86
src/datamodel/repository/tableMapLog.ts
Normal file
86
src/datamodel/repository/tableMapLog.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { Repository } from "./abstractRepository";
|
||||
import { v4 as uuid4 } from "uuid";
|
||||
|
||||
export interface TableMapLog {
|
||||
id: string;
|
||||
table_name: string;
|
||||
alias_name: string;
|
||||
source_file_name: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export class TableMapLogRepository extends Repository {
|
||||
|
||||
async init() {
|
||||
await this.createTable()
|
||||
}
|
||||
|
||||
async deleteMapping(id: string) {
|
||||
this.db.deleteData('table_map_log', [{ id: id }], 'id')
|
||||
}
|
||||
|
||||
private async createTable() {
|
||||
// FIXME: add autoincrement index here to make it easier to manage.
|
||||
await this.db.createTable('table_map_log', {
|
||||
'id': 'TEXT',
|
||||
'table_name': 'TEXT',
|
||||
'alias_name': 'TEXT',
|
||||
'source_file_name': 'TEXT',
|
||||
'created_at': 'TEXT',
|
||||
'updated_at': 'TEXT'
|
||||
}, true)
|
||||
}
|
||||
|
||||
async insert(log: Omit<TableMapLog, 'id' | 'created_at' | 'updated_at'>) {
|
||||
const now = Date.now()
|
||||
await this.db.insertData('table_map_log', [{
|
||||
id: uuid4(),
|
||||
table_name: log.table_name,
|
||||
alias_name: log.alias_name,
|
||||
source_file_name: log.source_file_name,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
}])
|
||||
}
|
||||
|
||||
async getAll() {
|
||||
const { data } = await this.db.select('SELECT * FROM table_map_log', {})
|
||||
return data as unknown as TableMapLog[]
|
||||
}
|
||||
|
||||
async getByAlias(sourceFileName: string, aliasName: string) {
|
||||
const { data } = await this.db.select(`SELECT * FROM table_map_log
|
||||
WHERE source_file_name=@source_file_name
|
||||
AND alias_name=@alias_name`, {
|
||||
'source_file_name': sourceFileName,
|
||||
'alias_name': aliasName
|
||||
})
|
||||
if (!data || data.length < 0) {
|
||||
return null
|
||||
}
|
||||
return data[0] as unknown as TableMapLog
|
||||
}
|
||||
|
||||
async getByTableName(tableName: string) {
|
||||
const { data } = await this.db.select(`SELECT * FROM table_map_log
|
||||
WHERE table_name = @table_name`, {
|
||||
table_name: tableName
|
||||
})
|
||||
if (!data) {
|
||||
return []
|
||||
}
|
||||
return data as unknown[] as TableMapLog[]
|
||||
}
|
||||
|
||||
|
||||
async getByContext(sourceFileName: string) {
|
||||
const { data } = await this.db.select(`SELECT * FROM table_map_log
|
||||
WHERE source_file_name=@source_file_name
|
||||
`, { source_file_name: sourceFileName })
|
||||
if (!data) {
|
||||
return []
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
199
src/datamodel/sync.ts
Normal file
199
src/datamodel/sync.ts
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
import { SqlSealDatabase } from "src/database/database";
|
||||
import { FileLog, FileLogRepository } from "./repository/fileLog";
|
||||
import { TableMapLogRepository } from "./repository/tableMapLog";
|
||||
import { TAbstractFile, TFile, Vault } from "obsidian";
|
||||
import { parse } from "papaparse";
|
||||
import { sanitise } from "src/utils/sanitiseColumn";
|
||||
import { FieldTypes, toTypeStatements } from "src/utils/typePredictions";
|
||||
import { FilepathHasher } from "../utils/hasher";
|
||||
import { Omnibus } from "@hypersphere/omnibus";
|
||||
|
||||
|
||||
interface TableRegistration {
|
||||
fileName: string;
|
||||
aliasName: string;
|
||||
sourceFile: string;
|
||||
}
|
||||
|
||||
export class Sync {
|
||||
private fileLog: FileLogRepository;
|
||||
private tableMapLog: TableMapLogRepository;
|
||||
private bus = new Omnibus()
|
||||
|
||||
private tableMaps = new Map<string, FileLog>()
|
||||
|
||||
constructor(
|
||||
private readonly db: SqlSealDatabase,
|
||||
private readonly vault: Vault
|
||||
) {
|
||||
|
||||
}
|
||||
|
||||
async updateTableMaps() {
|
||||
const data = await this.fileLog.getAll()
|
||||
this.tableMaps.clear()
|
||||
data.forEach(log => {
|
||||
this.tableMaps.set(log.file_name, log)
|
||||
})
|
||||
}
|
||||
|
||||
async init() {
|
||||
await this.db.connect()
|
||||
this.fileLog = new FileLogRepository(this.db)
|
||||
await this.fileLog.init()
|
||||
|
||||
this.tableMapLog = new TableMapLogRepository(this.db)
|
||||
await this.tableMapLog.init()
|
||||
|
||||
const fileLogs = await this.fileLog.getAll()
|
||||
for (const log of fileLogs) {
|
||||
await this.syncFileByName(log.file_name)
|
||||
}
|
||||
|
||||
await this.updateTableMaps()
|
||||
|
||||
// START SYNCING
|
||||
this.startSync()
|
||||
}
|
||||
|
||||
async syncFileByName(fileName: string) {
|
||||
const file = this.vault.getFileByPath(fileName)
|
||||
if (!file) {
|
||||
return
|
||||
}
|
||||
await this.syncFile(file)
|
||||
}
|
||||
|
||||
async syncFile(file: TFile) {
|
||||
if (!this.tableMaps.has(file.path)) {
|
||||
this.bus.trigger('file::change::'+file.path)
|
||||
return;
|
||||
}
|
||||
|
||||
const entry = this.tableMaps.get(file.path)!
|
||||
if (entry.file_hash !== file.stat.mtime.toString()) {
|
||||
const data = await this.vault.cachedRead(file)
|
||||
// TODO: PROBABLY SHOULD BE EXTRACTED SOMEWHERE FROM HERE later.
|
||||
const parsed = parse<Record<string, string>>(data, {
|
||||
header: true,
|
||||
dynamicTyping: false,
|
||||
skipEmptyLines: true,
|
||||
transformHeader: sanitise
|
||||
})
|
||||
const typeStatements = toTypeStatements(parsed.meta.fields ?? [], parsed.data)
|
||||
const columns = Object.entries(typeStatements.types).map(([key, value]) => ({
|
||||
name: key,
|
||||
type: value as FieldTypes
|
||||
}));
|
||||
const tableName = entry.table_name
|
||||
await this.db.createTableClean(tableName, columns)
|
||||
await this.db.insertData(tableName, typeStatements.data)
|
||||
await this.fileLog.updateHash(tableName, file.stat.mtime.toString())
|
||||
this.bus.trigger('change::' + tableName)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
startSync() {
|
||||
this.vault.on('modify', (file: TAbstractFile) => {
|
||||
if (file instanceof TFile) {
|
||||
this.syncFile(file)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async registerFile(log: Omit<FileLog, 'id' | 'created_at' | 'updated_at'>) {
|
||||
await this.fileLog.insert(log)
|
||||
await this.updateTableMaps()
|
||||
}
|
||||
|
||||
async getTablesMappingForContext(sourceFileName: string) {
|
||||
const tables = await this.tableMapLog.getByContext(sourceFileName)
|
||||
const map = tables.reduce((acc, t) => ({
|
||||
...acc,
|
||||
[t.alias_name as string]: t.table_name
|
||||
}), {})
|
||||
|
||||
return {
|
||||
...map,
|
||||
files: 'files',
|
||||
tasks: 'tasks',
|
||||
tags: 'tags'
|
||||
}
|
||||
}
|
||||
|
||||
async generateTableName(fileName: string) {
|
||||
const hash = await FilepathHasher.sha256(fileName)
|
||||
return `file_${hash}`
|
||||
}
|
||||
|
||||
async registerTable(reg: TableRegistration) {
|
||||
const existingFileLog = await this.fileLog.getByFilename(reg.fileName)
|
||||
let fileTableName: string;
|
||||
if (!existingFileLog) {
|
||||
// We need to create new one.
|
||||
const tableName = await this.generateTableName(reg.fileName)
|
||||
await this.registerFile({
|
||||
file_name: reg.fileName,
|
||||
table_name: tableName,
|
||||
file_hash: ''
|
||||
})
|
||||
fileTableName = tableName
|
||||
// INITIAL SYNC
|
||||
await this.syncFileByName(reg.fileName)
|
||||
} else {
|
||||
fileTableName = existingFileLog.table_name
|
||||
}
|
||||
|
||||
const existingTableLog = await this.tableMapLog.getByAlias(reg.sourceFile, reg.aliasName)
|
||||
if (!existingTableLog) {
|
||||
// Create new one
|
||||
await this.tableMapLog.insert({
|
||||
alias_name: reg.aliasName,
|
||||
source_file_name: reg.sourceFile,
|
||||
table_name: fileTableName,
|
||||
})
|
||||
} else {
|
||||
// Already exists, doing nothing.
|
||||
// Check if it is the same mapping
|
||||
if (existingTableLog.table_name != fileTableName) {
|
||||
await this.tableMapLog.deleteMapping(existingTableLog.id) // FIXME: might need to remove event too
|
||||
await this.tableMapLog.insert({
|
||||
alias_name: reg.aliasName,
|
||||
table_name: fileTableName,
|
||||
source_file_name: reg.sourceFile
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getRegistrator() {
|
||||
return this.bus.getRegistrator()
|
||||
}
|
||||
|
||||
async getEventNameForAlias(sourceFileName: string, aliasName: string) {
|
||||
const log = await this.tableMapLog.getByAlias(sourceFileName, aliasName)
|
||||
if (!log) {
|
||||
throw new Error(`${aliasName} does not exist for ${sourceFileName}`)
|
||||
}
|
||||
return `change::${log.table_name}`
|
||||
}
|
||||
|
||||
// async onFileChange(file: TFile) {
|
||||
// const path = file.path
|
||||
// const hash = file.stat.mtime
|
||||
// // Checking if the file is registered
|
||||
// const reg = await this.fileLog.getByFilename(path)
|
||||
// if (reg) {
|
||||
// // Need to syncronise it.
|
||||
// // Need to save update in the database.
|
||||
// // Need to trigger all related files.
|
||||
// const allRelated = this.tableMapLog.getByTableName(reg.tableName)
|
||||
// for (const related of allRelated) {
|
||||
// this.bus.bus.trigger('')
|
||||
// }
|
||||
// }
|
||||
|
||||
// // Checking if the file file is in the map.
|
||||
// }
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
import { App, MarkdownPostProcessorContext } from "obsidian"
|
||||
|
||||
export const resolveFrontmatter = (ctx: Pick<MarkdownPostProcessorContext, 'frontmatter' | 'sourcePath'>, app: App) => {
|
||||
if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) {
|
||||
return ctx.frontmatter as Record<string, any>
|
||||
}
|
||||
const file = app.vault.getFileByPath(ctx.sourcePath)
|
||||
if (!file) {
|
||||
return null
|
||||
}
|
||||
return app.metadataCache.getFileCache(file)?.frontmatter
|
||||
}
|
||||
118
src/hash.ts
118
src/hash.ts
|
|
@ -1,118 +0,0 @@
|
|||
|
||||
export function hashString(inputString: string): string {
|
||||
// Function to convert a number to alphanumeric character
|
||||
function toAlphanumeric(num: number): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
|
||||
return chars[num % chars.length];
|
||||
}
|
||||
|
||||
// Function to generate SHA-256 hash
|
||||
function sha256(input: string): string {
|
||||
// Simple SHA-256 implementation
|
||||
function sha256Internal(input: string): string {
|
||||
function rightRotate(value: number, shift: number): number {
|
||||
return (value >>> shift) | (value << (32 - shift));
|
||||
}
|
||||
|
||||
function toHexString(num: number): string {
|
||||
let hex = '';
|
||||
for (let i = 0; i < 8; i++) {
|
||||
hex += ((num >>> (24 - i * 4)) & 0xf).toString(16);
|
||||
}
|
||||
return hex;
|
||||
}
|
||||
|
||||
const words: number[] = [];
|
||||
for (let i = 0; i < input.length * 8; i += 8) {
|
||||
words[i >> 5] |= (input.charCodeAt(i / 8) & 0xff) << (24 - i % 32);
|
||||
}
|
||||
words[input.length >> 2] |= 0x80 << (24 - (input.length % 4) * 8);
|
||||
words[((input.length + 64 >> 9) << 4) + 15] = input.length * 8;
|
||||
|
||||
let a = 0x6a09e667;
|
||||
let b = 0xbb67ae85;
|
||||
let c = 0x3c6ef372;
|
||||
let d = 0xa54ff53a;
|
||||
let e = 0x510e527f;
|
||||
let f = 0x9b05688c;
|
||||
let g = 0x1f83d9ab;
|
||||
let h = 0x5be0cd19;
|
||||
|
||||
for (let i = 0; i < words.length; i += 16) {
|
||||
let aa = a;
|
||||
let bb = b;
|
||||
let cc = c;
|
||||
let dd = d;
|
||||
let ee = e;
|
||||
let ff = f;
|
||||
let gg = g;
|
||||
let hh = h;
|
||||
|
||||
for (let j = 0; j < 64; j++) {
|
||||
let T1 = hh + ((ee & ff) | (~ee & gg)) + words[i + j] + K[j];
|
||||
T1 = T1 | 0;
|
||||
let T2 = rightRotate(e, 6) ^ rightRotate(e, 11) ^ rightRotate(e, 25);
|
||||
T2 = T2 | 0;
|
||||
let T3 = (e & f) ^ (~e & g);
|
||||
T3 = T3 | 0;
|
||||
let T4 = h + T1 + T2 + T3 + H[j];
|
||||
T4 = T4 | 0;
|
||||
|
||||
h = g;
|
||||
g = f;
|
||||
f = e;
|
||||
e = d + T4;
|
||||
e = e | 0;
|
||||
d = c;
|
||||
c = b;
|
||||
b = a;
|
||||
a = T4 + T1;
|
||||
a = a | 0;
|
||||
}
|
||||
|
||||
a = a + aa;
|
||||
b = b + bb;
|
||||
c = c + cc;
|
||||
d = d + dd;
|
||||
e = e + ee;
|
||||
f = f + ff;
|
||||
g = g + gg;
|
||||
h = h + hh;
|
||||
}
|
||||
|
||||
return toHexString(a) + toHexString(b) + toHexString(c) + toHexString(d) +
|
||||
toHexString(e) + toHexString(f) + toHexString(g) + toHexString(h);
|
||||
}
|
||||
|
||||
// Convert the SHA-256 hash to alphanumeric characters
|
||||
function hashToAlphanumeric(hash: string): string {
|
||||
return hash.replace(/[^\w]/g, '');
|
||||
}
|
||||
|
||||
return hashToAlphanumeric(sha256Internal(input));
|
||||
}
|
||||
|
||||
// Constants for SHA-256
|
||||
const K = [
|
||||
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
||||
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
||||
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
||||
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
||||
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
||||
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
||||
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
||||
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
|
||||
];
|
||||
|
||||
const H = [
|
||||
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
|
||||
];
|
||||
|
||||
return sha256(inputString);
|
||||
}
|
||||
|
||||
|
||||
|
||||
export const generatePrefix = (prefix: string, tableName: string) => {
|
||||
return `TTT${prefix}_${tableName}`
|
||||
}
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
import { Menu, MenuItem, Plugin, TAbstractFile, Tasks, TFile } from 'obsidian';
|
||||
import { FilesFileSyncTable } from 'src/fileSyncTable/filesTable';
|
||||
import { TagsFileSyncTable } from 'src/fileSyncTable/tagsTable';
|
||||
import { TasksFileSyncTable } from 'src/fileSyncTable/tasksTable';
|
||||
import { Menu, Plugin, TAbstractFile, Tasks, TFile } from 'obsidian';
|
||||
import { GridRenderer } from 'src/renderer/GridRenderer';
|
||||
import { MarkdownRenderer } from 'src/renderer/MarkdownRenderer';
|
||||
import { TableRenderer } from 'src/renderer/TableRenderer';
|
||||
import { RendererConfig, RendererRegistry } from 'src/rendererRegistry';
|
||||
import { SealFileSync } from 'src/SealFileSync';
|
||||
import { RendererConfig, RendererRegistry } from 'src/renderer/rendererRegistry';
|
||||
import { DEFAULT_SETTINGS, SQLSealSettings, SQLSealSettingsTab } from 'src/settings/SQLSealSettingsTab';
|
||||
import { SqlSeal } from 'src/sqlSeal';
|
||||
import { SealFileSync } from 'src/vaultSync/SealFileSync';
|
||||
import { FilesFileSyncTable } from 'src/vaultSync/tables/filesTable';
|
||||
import { TagsFileSyncTable } from 'src/vaultSync/tables/tagsTable';
|
||||
import { TasksFileSyncTable } from 'src/vaultSync/tables/tasksTable';
|
||||
import { CSV_VIEW_TYPE, CSVView } from 'src/view/CSVView';
|
||||
|
||||
const GLOBAL_KEY = 'sqlSealApi'
|
||||
|
|
@ -37,18 +37,22 @@ export default class SqlSealPlugin extends Plugin {
|
|||
|
||||
this.registerGlobalApi();
|
||||
const sqlSeal = new SqlSeal(this.app, false, this.rendererRegistry) // FIXME: set verbose based on the env.
|
||||
this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler())
|
||||
this.sqlSeal = sqlSeal
|
||||
|
||||
await this.sqlSeal.db.connect()
|
||||
|
||||
// start syncing when files are loaded
|
||||
this.app.workspace.onLayoutReady(() => {
|
||||
sqlSeal.db.connect().then(() => {
|
||||
|
||||
this.fileSync = new SealFileSync(this.app, this)
|
||||
|
||||
this.fileSync.addTablePlugin(new FilesFileSyncTable(sqlSeal.db, this.app, sqlSeal.tablesManager, this))
|
||||
this.fileSync.addTablePlugin(new TagsFileSyncTable(sqlSeal.db, this.app, sqlSeal.tablesManager))
|
||||
this.fileSync.addTablePlugin(new TasksFileSyncTable(sqlSeal.db, this.app, sqlSeal.tablesManager))
|
||||
this.fileSync.addTablePlugin(new FilesFileSyncTable(sqlSeal.db, this.app, this))
|
||||
this.fileSync.addTablePlugin(new TagsFileSyncTable(sqlSeal.db, this.app))
|
||||
this.fileSync.addTablePlugin(new TasksFileSyncTable(sqlSeal.db, this.app))
|
||||
|
||||
this.fileSync.init()
|
||||
this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler())
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
import { SqlSealDatabase } from "src/database";
|
||||
|
||||
|
||||
export interface SyncEntry {
|
||||
filename: string;
|
||||
url: string;
|
||||
name: string;
|
||||
syncedAt: string;
|
||||
metadata: any;
|
||||
}
|
||||
|
||||
|
||||
export class SyncModel {
|
||||
tableName = 'sqlseal_sync'
|
||||
constructor(private db: SqlSealDatabase) {
|
||||
this.db.connect().then(() =>
|
||||
this.createTableIfNotExists()
|
||||
)
|
||||
}
|
||||
|
||||
createTableIfNotExists() {
|
||||
this.db.createTable(this.tableName, {
|
||||
'filename': 'TEXT',
|
||||
'checksum': 'TEXT',
|
||||
'syncedAt': 'TEXT',
|
||||
})
|
||||
}
|
||||
|
||||
registerSync(filename: string, checksum: string) {
|
||||
this.db.insertData(this.tableName, [{
|
||||
filename,
|
||||
checksum,
|
||||
syncedAt: Date.now(),
|
||||
}])
|
||||
}
|
||||
|
||||
async getSync(filename: string) {
|
||||
const { data } = await this.db.select(`SELECT *
|
||||
FROM sqlseal_sync
|
||||
WHERE filename = @filename`,
|
||||
{
|
||||
filename: filename
|
||||
})
|
||||
|
||||
if (data) {
|
||||
return data[0] as unknown as SyncEntry
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
removeSync(filename: string) {
|
||||
// FIXME: do not use exposed db, instead implement "exec" method inside the file.
|
||||
this.db.db.prepare('DELETE FROM sqlseal_sync WHERE filename = :filename AND name = :name').run({
|
||||
filename: filename,
|
||||
name: tableName
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import { createGrid, GridApi, GridOptions, themeQuartz } from "ag-grid-community";
|
||||
import { merge } from "lodash";
|
||||
import { App } from "obsidian";
|
||||
import { RendererConfig } from "src/rendererRegistry";
|
||||
import { RendererConfig } from "src/renderer/rendererRegistry";
|
||||
import { parse } from 'json5'
|
||||
import { displayError, displayNotice, parseCell } from "src/ui";
|
||||
import { parseCell } from "src/utils/ui";
|
||||
|
||||
const getCurrentTheme = () => {
|
||||
return document.body.classList.contains('theme-dark') ? 'dark' : 'light';
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
// This is renderer for a very basic Table view.
|
||||
import { getMarkdownTable } from "markdown-table-ts";
|
||||
import { App } from "obsidian";
|
||||
import { RendererConfig } from "src/rendererRegistry";
|
||||
import { displayError, parseCell } from "src/ui";
|
||||
import { RendererConfig } from "src/renderer/rendererRegistry";
|
||||
import { displayError, parseCell } from "src/utils/ui";
|
||||
|
||||
const mapDataFromHeaders = (columns: string[], data: Record<string, any>[]) => {
|
||||
return data.map(d => columns.map(c => String(d[c])))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// This is renderer for a very basic Table view.
|
||||
import { App } from "obsidian";
|
||||
import { RendererConfig } from "src/rendererRegistry";
|
||||
import { displayError, parseCell } from "src/ui";
|
||||
import { RendererConfig } from "src/renderer/rendererRegistry";
|
||||
import { displayError, parseCell } from "src/utils/ui";
|
||||
|
||||
export class TableRenderer implements RendererConfig {
|
||||
|
||||
|
|
|
|||
|
|
@ -26,10 +26,6 @@ export class SQLSealSettingsTab extends PluginSettingTab {
|
|||
const {containerEl} = this;
|
||||
containerEl.empty();
|
||||
|
||||
// CSV Viewer section
|
||||
// containerEl.createEl('h3', { text: 'CSV Viewer' });
|
||||
|
||||
|
||||
new Setting(containerEl)
|
||||
.setName('Enable CSV Viewer')
|
||||
.setDesc('Enables CSV files in your vault and adds ability to display them in a grid.')
|
||||
|
|
|
|||
23
src/sql/transformer.test.ts
Normal file
23
src/sql/transformer.test.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import { describe, it, expect } from '@jest/globals'
|
||||
import { transformQuery } from './transformer'
|
||||
|
||||
|
||||
describe('SQL Transformer', () => {
|
||||
it('should properly transform basic SQL query', () => {
|
||||
const transformed = transformQuery('SELECT * FROM x', { x: 'file_123' })
|
||||
expect(transformed).toEqual('SELECT * FROM "file_123"')
|
||||
})
|
||||
|
||||
it('should properly transform SQL with CTE', () => {
|
||||
const transformed = transformQuery('WITH virtual AS (SELECT * FROM x) SELECT * FROM virtual, x', { x: 'y' })
|
||||
expect(transformed).toEqual("WITH \"virtual\" AS (SELECT * FROM \"y\") SELECT * FROM \"virtual\", \"y\"")
|
||||
})
|
||||
|
||||
it('should properly persist table functions like json_each', () => {
|
||||
const transformed = transformQuery(`SELECT y.name, x.value
|
||||
FROM y, json_each(y.list) as x
|
||||
WHERE list IS NOT NULL`, { y: 'file_123' })
|
||||
|
||||
expect(transformed).toEqual("SELECT \"file_123\".\"name\", \"x\".\"value\" FROM \"file_123\", json_each(\"file_123\".\"list\") AS \"x\" WHERE \"list\" IS NOT NULL")
|
||||
})
|
||||
})
|
||||
143
src/sql/transformer.ts
Normal file
143
src/sql/transformer.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
import { Parser } from 'node-sql-parser';
|
||||
|
||||
export function transformQuery(query: string, tablesAliases: Record<string, string>): string {
|
||||
const parser = new Parser();
|
||||
|
||||
// Parse the query into an AST
|
||||
const ast = parser.astify(query, {
|
||||
database: 'Sqlite'
|
||||
});
|
||||
|
||||
// Handle both single queries and multiple queries (like UNION)
|
||||
const queries = Array.isArray(ast) ? ast : [ast];
|
||||
|
||||
// Process each query in the array
|
||||
queries.forEach(queryAst => {
|
||||
transformQueryPart(queryAst, tablesAliases);
|
||||
});
|
||||
|
||||
// Convert back to SQL
|
||||
return parser.sqlify(ast, { database: 'Sqlite' });
|
||||
}
|
||||
|
||||
function transformQueryPart(ast: any, tablesAliases: Record<string, string>): void {
|
||||
if (!ast || typeof ast !== 'object') return;
|
||||
|
||||
// Handle FROM clause
|
||||
if (ast.from) {
|
||||
ast.from = ast.from.map((fromItem: any) => {
|
||||
// Handle regular table references
|
||||
if (fromItem.table && tablesAliases[fromItem.table]) {
|
||||
fromItem.table = tablesAliases[fromItem.table];
|
||||
}
|
||||
|
||||
// Handle function expressions in FROM clause (like json_each)
|
||||
if (fromItem.expr) {
|
||||
transformExpression(fromItem.expr, tablesAliases);
|
||||
}
|
||||
|
||||
// Handle nested queries in FROM
|
||||
if (fromItem.stmt) {
|
||||
transformQueryPart(fromItem.stmt, tablesAliases);
|
||||
}
|
||||
|
||||
return fromItem;
|
||||
});
|
||||
}
|
||||
|
||||
// Handle WITH clause (CTEs)
|
||||
if (ast.with && Array.isArray(ast.with)) {
|
||||
ast.with.forEach((withItem: any) => {
|
||||
if (withItem.stmt) {
|
||||
transformQueryPart(withItem.stmt.ast, tablesAliases);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle legacy WITH clause structure
|
||||
if (ast.with && ast.with.ctes && Array.isArray(ast.with.ctes)) {
|
||||
ast.with.ctes.forEach((withItem: any) => {
|
||||
if (withItem.stmt) {
|
||||
transformQueryPart(withItem.stmt, tablesAliases);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Handle JOINs
|
||||
if (ast.join) {
|
||||
ast.join = ast.join.map((joinItem: any) => {
|
||||
if (joinItem.table && joinItem.table.table && tablesAliases[joinItem.table.table]) {
|
||||
joinItem.table.table = tablesAliases[joinItem.table.table];
|
||||
}
|
||||
if (joinItem.on) {
|
||||
transformExpression(joinItem.on, tablesAliases);
|
||||
}
|
||||
return joinItem;
|
||||
});
|
||||
}
|
||||
|
||||
// Handle WHERE clause
|
||||
if (ast.where) {
|
||||
transformExpression(ast.where, tablesAliases);
|
||||
}
|
||||
|
||||
// Handle UNION queries
|
||||
if (ast.union) {
|
||||
transformQueryPart(ast.union[0], tablesAliases);
|
||||
transformQueryPart(ast.union[1], tablesAliases);
|
||||
}
|
||||
|
||||
// Handle SELECT columns
|
||||
if (ast.columns) {
|
||||
ast.columns.forEach((column: any) => {
|
||||
transformExpression(column.expr, tablesAliases);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function transformExpression(expr: any, tablesAliases: Record<string, string>): void {
|
||||
if (!expr || typeof expr !== 'object') return;
|
||||
|
||||
// Handle column references
|
||||
if (expr.type === 'column_ref' && expr.table && tablesAliases[expr.table]) {
|
||||
expr.table = tablesAliases[expr.table];
|
||||
}
|
||||
|
||||
// Handle function calls
|
||||
if (expr.type === 'function') {
|
||||
if (expr.args && Array.isArray(expr.args)) {
|
||||
expr.args.forEach((arg: any) => {
|
||||
if (arg.type === 'column_ref' && arg.table && tablesAliases[arg.table]) {
|
||||
arg.table = tablesAliases[arg.table];
|
||||
}
|
||||
transformExpression(arg, tablesAliases);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Handle binary expressions
|
||||
if (expr.type === 'binary_expr') {
|
||||
transformExpression(expr.left, tablesAliases);
|
||||
transformExpression(expr.right, tablesAliases);
|
||||
}
|
||||
|
||||
// Handle CASE expressions
|
||||
if (expr.type === 'case') {
|
||||
if (expr.args && Array.isArray(expr.args)) {
|
||||
expr.args.forEach((arg: any) => transformExpression(arg, tablesAliases));
|
||||
}
|
||||
}
|
||||
|
||||
// Recursively process other potential expressions
|
||||
Object.keys(expr).forEach(key => {
|
||||
if (Array.isArray(expr[key])) {
|
||||
expr[key].forEach((item: any) => {
|
||||
if (item && typeof item === 'object') {
|
||||
transformExpression(item, tablesAliases);
|
||||
}
|
||||
});
|
||||
} else if (expr[key] && typeof expr[key] === 'object') {
|
||||
transformExpression(expr[key], tablesAliases);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
import { BaseFrom, From, Parser, Select, With } from "node-sql-parser"
|
||||
import { generatePrefix } from "./hash"
|
||||
|
||||
const isGlobal = (table: string, globalTables: string[]) => {
|
||||
return globalTables.map(t =>
|
||||
t.toLowerCase()
|
||||
).includes(table.toLowerCase())
|
||||
}
|
||||
|
||||
const isBaseFrom = (from: From): from is BaseFrom => {
|
||||
return Object.keys(from).includes('table')
|
||||
}
|
||||
|
||||
export const prefixedIfNotGlobal = (tableName: string, globalTables: string[], prefix: string) => {
|
||||
if (isGlobal(tableName, globalTables)) {
|
||||
return tableName
|
||||
}
|
||||
|
||||
return generatePrefix(prefix, tableName)
|
||||
}
|
||||
|
||||
const updateSelect = (selectAst: Select, globalTables: string[], prefix: string): { selectAst: Select, tables: string[] } => {
|
||||
|
||||
let cteGlobals: string[] = []
|
||||
let withs = selectAst.with
|
||||
let extraTables: string[] = []
|
||||
if (selectAst.with) {
|
||||
cteGlobals = selectAst.with.map(w => w.name.value)
|
||||
const updatedWiths = selectAst.with.map(w => {
|
||||
const { selectAst, tables } = updateSelect(w.stmt.ast, [...cteGlobals, ...globalTables], prefix)
|
||||
extraTables.push(...tables)
|
||||
return {
|
||||
...w,
|
||||
stmt: {
|
||||
...w.stmt,
|
||||
ast: selectAst
|
||||
}
|
||||
} satisfies With
|
||||
})
|
||||
withs = updatedWiths
|
||||
}
|
||||
|
||||
if (!selectAst.from) {
|
||||
return { selectAst: selectAst, tables: [] }
|
||||
}
|
||||
|
||||
const tables: Array<string> = []
|
||||
|
||||
const updatedFrom = selectAst.from.map(from => {
|
||||
if(isBaseFrom(from)) {
|
||||
const t = prefixedIfNotGlobal(from.table, [...cteGlobals, ...globalTables], prefix)
|
||||
tables.push(t)
|
||||
return {
|
||||
...from,
|
||||
table: t,
|
||||
}
|
||||
}
|
||||
return from
|
||||
})
|
||||
|
||||
return {
|
||||
selectAst: {
|
||||
...selectAst,
|
||||
from: updatedFrom,
|
||||
with: withs
|
||||
},
|
||||
tables: [...tables, ...extraTables]
|
||||
}
|
||||
}
|
||||
|
||||
export const updateTables = (selectStatement: string, globalTables: string[], prefix: string) => {
|
||||
const parser = new Parser()
|
||||
const { ast } = parser.parse(selectStatement)
|
||||
if (!Array.isArray(ast) && ast.type === 'select') {
|
||||
const { selectAst, tables } = updateSelect(ast!, globalTables, prefix)
|
||||
return { statement: parser.sqlify(selectAst), tables }
|
||||
} else {
|
||||
throw new Error('Invalid Statement. Only single SELECTs are accepted at the moment.')
|
||||
}
|
||||
}
|
||||
|
||||
export const extractCtes = (selectStatement: string) => {
|
||||
const parser = new Parser()
|
||||
const { ast } = parser.parse(selectStatement)
|
||||
if (!Array.isArray(ast) && ast.type === 'select') {
|
||||
if (ast.with) {
|
||||
return ast.with.map(w => w.name.value)
|
||||
}
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
|
@ -1,28 +1,21 @@
|
|||
import { SqlSealDatabase } from "./database";
|
||||
import { SqlSealDatabase } from "./database/database";
|
||||
import { App } from "obsidian";
|
||||
import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler";
|
||||
import { Logger } from "./logger";
|
||||
import { TablesManager } from "./dataLoader/collections/tablesManager";
|
||||
import { QueryManager } from "./dataLoader/collections/queryManager";
|
||||
import { FilesManager } from "./dataLoader/collections/filesManager";
|
||||
import { RendererRegistry } from "./rendererRegistry";
|
||||
import { SqlSealCodeblockHandler } from "./codeblockHandler/SqlSealCodeblockHandler";
|
||||
import { Logger } from "./utils/logger";
|
||||
import { RendererRegistry } from "./renderer/rendererRegistry";
|
||||
import { Sync } from "./datamodel/sync";
|
||||
|
||||
export class SqlSeal {
|
||||
public db: SqlSealDatabase
|
||||
public codeBlockHandler: SqlSealCodeblockHandler
|
||||
public tablesManager: TablesManager
|
||||
constructor(private readonly app: App, verbose = false, rendererRegistry: RendererRegistry) {
|
||||
this.db = new SqlSealDatabase(app, verbose)
|
||||
const logger = new Logger(verbose)
|
||||
|
||||
const fileManager = new FilesManager(this.app.vault, this.app, this.db)
|
||||
this.tablesManager = new TablesManager(fileManager, this.db)
|
||||
this.tablesManager.getTableSignal('files')
|
||||
this.tablesManager.getTableSignal('tags')
|
||||
const queryManager = new QueryManager(this.tablesManager)
|
||||
const sync = new Sync(this.db, this.app.vault)
|
||||
sync.init()
|
||||
|
||||
this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, logger, this.tablesManager, queryManager, rendererRegistry)
|
||||
// FIXME: handle here changes to files and tags?
|
||||
this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, sync, rendererRegistry)
|
||||
}
|
||||
|
||||
getHandler() {
|
||||
|
|
|
|||
54
src/utils/hasher.ts
Normal file
54
src/utils/hasher.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/**
|
||||
* A utility class for generating consistent hashes from filepaths in browser environments
|
||||
*/
|
||||
export class FilepathHasher {
|
||||
/**
|
||||
* Creates a SHA-256 hash of a filepath
|
||||
* @param {string} filepath - The filepath to hash
|
||||
* @returns {Promise<string>} The hex-encoded hash
|
||||
*/
|
||||
static async sha256(filepath: string) {
|
||||
// Normalize the filepath to ensure consistent hashing across platforms
|
||||
const normalizedPath = filepath.replace(/\\/g, '/').toLowerCase();
|
||||
|
||||
// Convert string to array buffer
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(normalizedPath);
|
||||
|
||||
// Generate hash using browser's crypto API
|
||||
const hashBuffer = await window.crypto.subtle.digest('SHA-256', data);
|
||||
|
||||
// Convert to hex string
|
||||
return Array.from(new Uint8Array(hashBuffer))
|
||||
.map(b => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a shorter hash (first 8 characters of SHA-256)
|
||||
* @param {string} filepath - The filepath to hash
|
||||
* @returns {Promise<string>} The truncated hex-encoded hash
|
||||
*/
|
||||
static async shortHash(filepath: string) {
|
||||
const fullHash = await this.sha256(filepath);
|
||||
return fullHash.slice(0, 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a base64 encoded hash of a filepath
|
||||
* @param {string} filepath - The filepath to hash
|
||||
* @returns {Promise<string>} The base64-encoded hash
|
||||
*/
|
||||
static async base64Hash(filepath: string) {
|
||||
const normalizedPath = filepath.replace(/\\/g, '/').toLowerCase();
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(normalizedPath);
|
||||
const hashBuffer = await window.crypto.subtle.digest('SHA-256', data);
|
||||
|
||||
// Convert ArrayBuffer to Base64
|
||||
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
||||
const hashString = String.fromCharCode.apply(null, hashArray);
|
||||
return btoa(hashString);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1,247 +0,0 @@
|
|||
import { describe, it, expect, jest } from '@jest/globals'
|
||||
import { createSignal, derivedSignal, withSignals } from './signal';
|
||||
|
||||
describe('createSignal', () => {
|
||||
it('should create a signal with initial value', () => {
|
||||
const signal = createSignal(10);
|
||||
expect(signal.value).toBe(10);
|
||||
});
|
||||
|
||||
it('should create a signal without initial value', () => {
|
||||
const signal = createSignal<number>();
|
||||
expect(signal.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should notify subscribers when value changes', () => {
|
||||
const signal = createSignal<number>();
|
||||
const listener = jest.fn();
|
||||
|
||||
signal(listener);
|
||||
signal(42);
|
||||
|
||||
expect(listener).toHaveBeenCalledWith(42);
|
||||
});
|
||||
|
||||
it('should call listener immediately with current value if exists', () => {
|
||||
const signal = createSignal(10);
|
||||
const listener = jest.fn();
|
||||
|
||||
signal(listener);
|
||||
expect(listener).toHaveBeenCalledWith(10);
|
||||
});
|
||||
|
||||
it('should allow unsubscribing', () => {
|
||||
const signal = createSignal<number>();
|
||||
const listener = jest.fn();
|
||||
|
||||
const unsubscribe = signal(listener);
|
||||
signal(42);
|
||||
expect(listener).toHaveBeenCalledTimes(1);
|
||||
|
||||
unsubscribe();
|
||||
signal(43);
|
||||
expect(listener).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should allow multiple subscribers', () => {
|
||||
const signal = createSignal<number>();
|
||||
const listener1 = jest.fn();
|
||||
const listener2 = jest.fn();
|
||||
|
||||
signal(listener1);
|
||||
signal(listener2);
|
||||
signal(42);
|
||||
|
||||
expect(listener1).toHaveBeenCalledWith(42);
|
||||
expect(listener2).toHaveBeenCalledWith(42);
|
||||
});
|
||||
});
|
||||
|
||||
describe('withSignals', () => {
|
||||
it('should combine multiple signals', () => {
|
||||
const signal1 = createSignal<number>();
|
||||
const signal2 = createSignal<string>();
|
||||
const callback = jest.fn();
|
||||
|
||||
withSignals(signal1, signal2)(callback);
|
||||
|
||||
signal1(42);
|
||||
expect(callback).not.toHaveBeenCalled();
|
||||
|
||||
signal2('hello');
|
||||
expect(callback).toHaveBeenCalledWith(42, 'hello');
|
||||
});
|
||||
|
||||
it('should work with initial values', () => {
|
||||
const signal1 = createSignal(42);
|
||||
const signal2 = createSignal('hello');
|
||||
const callback = jest.fn();
|
||||
|
||||
withSignals(signal1, signal2)(callback);
|
||||
expect(callback).toHaveBeenCalledWith(42, 'hello');
|
||||
});
|
||||
|
||||
it('should update when any signal changes', () => {
|
||||
const signal1 = createSignal(42);
|
||||
const signal2 = createSignal('hello');
|
||||
const callback = jest.fn();
|
||||
|
||||
withSignals(signal1, signal2)(callback);
|
||||
signal1(43);
|
||||
|
||||
expect(callback).toHaveBeenCalledWith(43, 'hello');
|
||||
});
|
||||
|
||||
it('should allow unsubscribing from all signals', () => {
|
||||
const signal1 = createSignal<number>();
|
||||
const signal2 = createSignal<string>();
|
||||
const callback = jest.fn();
|
||||
|
||||
const unsubscribe = withSignals(signal1, signal2)(callback);
|
||||
|
||||
signal1(42);
|
||||
signal2('hello');
|
||||
expect(callback).toHaveBeenCalledTimes(1);
|
||||
|
||||
unsubscribe();
|
||||
signal1(43);
|
||||
signal2('world');
|
||||
expect(callback).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('derivedSignal', () => {
|
||||
it('should compute derived value', () => {
|
||||
const count = createSignal(5);
|
||||
const multiplier = createSignal(2);
|
||||
|
||||
const product = derivedSignal(
|
||||
[count, multiplier],
|
||||
(c, m) => c * m
|
||||
);
|
||||
|
||||
expect(product.value).toBe(10);
|
||||
});
|
||||
|
||||
it('should update when source signals change', () => {
|
||||
const count = createSignal(5);
|
||||
const multiplier = createSignal(2);
|
||||
|
||||
const product = derivedSignal(
|
||||
[count, multiplier],
|
||||
(c, m) => c * m
|
||||
);
|
||||
|
||||
const listener = jest.fn();
|
||||
product(listener);
|
||||
|
||||
count(10);
|
||||
expect(product.value).toBe(20);
|
||||
expect(listener).toHaveBeenCalledWith(20);
|
||||
|
||||
multiplier(3);
|
||||
expect(product.value).toBe(30);
|
||||
expect(listener).toHaveBeenCalledWith(30);
|
||||
});
|
||||
|
||||
it('should handle undefined initial values', () => {
|
||||
const signal1 = createSignal<number>();
|
||||
const signal2 = createSignal<number>();
|
||||
|
||||
const sum = derivedSignal(
|
||||
[signal1, signal2],
|
||||
(a, b) => a + b
|
||||
);
|
||||
|
||||
expect(sum.value).toBeUndefined();
|
||||
|
||||
signal1(5);
|
||||
expect(sum.value).toBeUndefined();
|
||||
|
||||
signal2(3);
|
||||
expect(sum.value).toBe(8);
|
||||
});
|
||||
|
||||
it('should work with complex derivations', () => {
|
||||
const firstName = createSignal('John');
|
||||
const lastName = createSignal('Doe');
|
||||
const age = createSignal(25);
|
||||
|
||||
const person = derivedSignal(
|
||||
[firstName, lastName, age],
|
||||
(first, last, a) => ({
|
||||
fullName: `${first} ${last}`,
|
||||
age: a,
|
||||
isAdult: a >= 18
|
||||
})
|
||||
);
|
||||
|
||||
expect(person.value).toEqual({
|
||||
fullName: 'John Doe',
|
||||
age: 25,
|
||||
isAdult: true
|
||||
});
|
||||
|
||||
firstName('Jane');
|
||||
expect(person.value).toEqual({
|
||||
fullName: 'Jane Doe',
|
||||
age: 25,
|
||||
isAdult: true
|
||||
});
|
||||
});
|
||||
|
||||
it('should chain derived signals', () => {
|
||||
const base = createSignal(5);
|
||||
const doubled = derivedSignal([base], n => n * 2);
|
||||
const final = derivedSignal([doubled], n => n + 10);
|
||||
|
||||
expect(final.value).toBe(20);
|
||||
|
||||
base(10);
|
||||
expect(doubled.value).toBe(20);
|
||||
expect(final.value).toBe(30);
|
||||
});
|
||||
|
||||
it('should support array transformations', () => {
|
||||
const items = createSignal(['apple', 'banana', 'orange']);
|
||||
const filter = createSignal('an');
|
||||
|
||||
const filtered = derivedSignal(
|
||||
[items, filter],
|
||||
(list, f) => list.filter(item =>
|
||||
item.toLowerCase().includes(f.toLowerCase())
|
||||
)
|
||||
);
|
||||
|
||||
expect(filtered.value).toEqual(['banana', 'orange']);
|
||||
|
||||
filter('ap');
|
||||
expect(filtered.value).toEqual(['apple']);
|
||||
|
||||
items(['grape', 'apple', 'mango']);
|
||||
expect(filtered.value).toEqual(['grape', 'apple']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('type safety', () => {
|
||||
it('should maintain proper types in derived signals', () => {
|
||||
const numberSignal = createSignal<number>(42);
|
||||
const stringSignal = createSignal<string>('hello');
|
||||
|
||||
// This should type check
|
||||
const derived = derivedSignal(
|
||||
[numberSignal, stringSignal],
|
||||
(num, str) => ({
|
||||
number: num, // Should be typed as number
|
||||
string: str, // Should be typed as string
|
||||
combined: `${str}${num}`
|
||||
})
|
||||
);
|
||||
|
||||
expect(derived.value).toEqual({
|
||||
number: 42,
|
||||
string: 'hello',
|
||||
combined: 'hello42'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,145 +0,0 @@
|
|||
/**
|
||||
* Enhanced signal types to include value access
|
||||
*/
|
||||
export interface SignalListener<T> {
|
||||
(event: T): void;
|
||||
}
|
||||
|
||||
export interface SignalUnsubscriber {
|
||||
(): void;
|
||||
}
|
||||
|
||||
export interface SignalSubscriber<T> {
|
||||
(listener: SignalListener<T>): SignalUnsubscriber;
|
||||
}
|
||||
|
||||
export interface SignalDispatcher<T> {
|
||||
(event: T): void;
|
||||
}
|
||||
|
||||
export interface SignalValue<T> {
|
||||
readonly value: T;
|
||||
}
|
||||
|
||||
export type Signal<T> = SignalSubscriber<T> & SignalDispatcher<T> & SignalValue<T>;
|
||||
export type SignalReturn = SignalUnsubscriber & void;
|
||||
|
||||
export function createSignal<T>(): Signal<T>;
|
||||
export function createSignal<T>(initialValue: T): Signal<T>;
|
||||
export function createSignal<T>(initialValue?: T): Signal<T> {
|
||||
const subscribers = new Set<SignalListener<T>>();
|
||||
let currentValue: T | undefined = initialValue;
|
||||
|
||||
const signal = ((eventOrListener: T | SignalListener<T>): any => {
|
||||
if (typeof eventOrListener === 'function') {
|
||||
subscribers.add(eventOrListener as SignalListener<T>);
|
||||
// Call the listener immediately with current value if it exists
|
||||
if (currentValue !== undefined) {
|
||||
(eventOrListener as SignalListener<T>)(currentValue);
|
||||
}
|
||||
return () => { subscribers.delete(eventOrListener as SignalListener<T>) };
|
||||
} else {
|
||||
currentValue = eventOrListener;
|
||||
subscribers.forEach(listener => listener(eventOrListener));
|
||||
}
|
||||
}) as Signal<T>;
|
||||
|
||||
// Add value getter
|
||||
Object.defineProperty(signal, 'value', {
|
||||
get: () => currentValue,
|
||||
enumerable: true
|
||||
});
|
||||
|
||||
return signal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility types for handling async compute functions
|
||||
*/
|
||||
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T;
|
||||
|
||||
type ComputeReturnType<Args extends any[], R> =
|
||||
| ((...args: Args) => R)
|
||||
| ((...args: Args) => Promise<R>);
|
||||
|
||||
/**
|
||||
* Utility type to extract the event type from a Signal
|
||||
*/
|
||||
export type SignalEventType<T> = T extends Signal<infer E> ? E : never;
|
||||
|
||||
/**
|
||||
* Creates a new signal that derives its value from other signals
|
||||
* Supports both synchronous and asynchronous compute functions
|
||||
*/
|
||||
export function derivedSignal<Signals extends Signal<any>[], R>(
|
||||
signals: [...Signals],
|
||||
compute: ComputeReturnType<{ [K in keyof Signals]: SignalEventType<Signals[K]> }, R>
|
||||
): Signal<UnwrapPromise<R>> {
|
||||
const derivedSignal = createSignal<UnwrapPromise<R>>();
|
||||
let computeVersion = 0;
|
||||
let isComputing = false;
|
||||
|
||||
const updateValue = async (version: number, values: { [K in keyof Signals]: SignalEventType<Signals[K]> }) => {
|
||||
if (isComputing) return;
|
||||
isComputing = true;
|
||||
|
||||
try {
|
||||
const result = compute(...Object.values(values) as { [K in keyof Signals]: SignalEventType<Signals[K]> });
|
||||
const computedValue = result instanceof Promise ? await result : result;
|
||||
|
||||
// Only update if this is still the latest computation
|
||||
if (version === computeVersion) {
|
||||
derivedSignal(computedValue as UnwrapPromise<R>);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error in derived signal computation:', error);
|
||||
} finally {
|
||||
isComputing = false;
|
||||
}
|
||||
};
|
||||
|
||||
// Compute initial value if all source signals have values
|
||||
const initialValues = signals.map(s => s.value);
|
||||
if (!initialValues.includes(undefined)) {
|
||||
updateValue(computeVersion, initialValues as { [K in keyof Signals]: SignalEventType<Signals[K]> });
|
||||
}
|
||||
|
||||
// Handle updates
|
||||
withSignals(...signals)((...args) => {
|
||||
computeVersion++;
|
||||
updateValue(computeVersion, args as { [K in keyof Signals]: SignalEventType<Signals[K]> });
|
||||
});
|
||||
|
||||
return derivedSignal;
|
||||
}
|
||||
/**
|
||||
* Takes multiple signals and returns a function that accepts a callback
|
||||
* which will receive the values from those signals
|
||||
*/
|
||||
export function withSignals<Signals extends Signal<any>[]>(
|
||||
...signals: Signals
|
||||
): <R>(
|
||||
callback: (...values: { [K in keyof Signals]: SignalEventType<Signals[K]> }) => R
|
||||
) => SignalUnsubscriber {
|
||||
return (callback) => {
|
||||
const unsubscribers: SignalUnsubscriber[] = [];
|
||||
const values = new Array(signals.length) as { [K in keyof Signals]: SignalEventType<Signals[K]> };
|
||||
let initialized = new Array(signals.length).fill(false);
|
||||
|
||||
signals.forEach((signal, index) => {
|
||||
const unsubscribe = signal((value) => {
|
||||
values[index] = value as { [K in keyof Signals]: SignalEventType<Signals[K]> }[number];
|
||||
initialized[index] = true;
|
||||
|
||||
if (initialized.every(Boolean)) {
|
||||
callback(...(values as unknown as { [K in keyof Signals]: SignalEventType<Signals[K]> }));
|
||||
}
|
||||
});
|
||||
unsubscribers.push(unsubscribe);
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubscribers.forEach(unsubscribe => unsubscribe());
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,8 +1,7 @@
|
|||
import { describe, it, expect } from '@jest/globals'
|
||||
import { predictJson, predictType } from './utils'
|
||||
import { predictJson, predictType } from './typePredictions'
|
||||
|
||||
describe('Utils', () => {
|
||||
|
||||
it('should properly predict TEXT type', () => {
|
||||
expect(predictType('xxx', [{ 'xxx': 'dhjkafhkjafhkfas' }])).toEqual('TEXT')
|
||||
})
|
||||
|
|
@ -1,56 +1,7 @@
|
|||
|
||||
import { get } from "https"
|
||||
import { parse } from 'json5'
|
||||
import * as fs from 'fs'
|
||||
import { camelCase } from "lodash";
|
||||
|
||||
export const fetchBlobData = async (url: string, filePath: string) => {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
get(url, (response) => {
|
||||
const statusCode = response.statusCode;
|
||||
const contentType = response.headers['content-type'];
|
||||
|
||||
let error;
|
||||
if (statusCode === 301 || statusCode === 302) {
|
||||
// Handle redirect
|
||||
const redirectUrl = response.headers.location;
|
||||
if (!redirectUrl) {
|
||||
reject('Redirect URL not found')
|
||||
return;
|
||||
}
|
||||
fetchBlobData(redirectUrl, filePath).then(resolve).catch(reject);
|
||||
return;
|
||||
}
|
||||
if (statusCode !== 200) {
|
||||
error = new Error(`Request Failed. Status Code: ${statusCode}`);
|
||||
} else if (!/^application\/octet-stream/.test(contentType ?? '')) {
|
||||
error = new Error(`Invalid content-type. Expected application/octet-stream but received ${contentType}`);
|
||||
}
|
||||
if (error) {
|
||||
console.error(error.message);
|
||||
// Consume response data to free up memory
|
||||
response.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
const fileStream = fs.createWriteStream(filePath);
|
||||
response.pipe(fileStream);
|
||||
|
||||
fileStream.on('finish', () => {
|
||||
resolve()
|
||||
fileStream.close()
|
||||
});
|
||||
|
||||
fileStream.on('error', (err) => {
|
||||
console.error(`Error writing to ${filePath}: ${err}`);
|
||||
reject(`Error writing to ${filePath}: ${err}`)
|
||||
});
|
||||
}).on('error', (err) => {
|
||||
console.error(`Error fetching blob data: ${err.message}`);
|
||||
reject(`Error writing to ${filePath}: ${err}`)
|
||||
});
|
||||
})
|
||||
};
|
||||
export const predictType = (field: string, data: Array<Record<string, string | Object>>) => {
|
||||
|
||||
if (field === 'id') {
|
||||
|
|
@ -1,6 +1,4 @@
|
|||
import { App } from "obsidian"
|
||||
import { createGrid, GridOptions } from 'ag-grid-community';
|
||||
import { themeQuartz } from '@ag-grid-community/theming';
|
||||
|
||||
export const displayNotice = (el: HTMLElement, text: string) => {
|
||||
el.empty()
|
||||
|
|
@ -51,7 +49,6 @@ const generateLink = (config: SqlSealAnchorElement, app: App) => {
|
|||
|
||||
link.addEventListener('click', (event) => {
|
||||
event.preventDefault();
|
||||
|
||||
// Open the file in the active leaf (same tab)
|
||||
const leaf = app.workspace.getLeaf();
|
||||
const file = app.vault.getFileByPath(config.href)
|
||||
|
|
@ -1,18 +1,7 @@
|
|||
import { App, Plugin, TAbstractFile, TFile } from "obsidian";
|
||||
import { SqlSeal } from "./sqlSeal";
|
||||
import { FieldTypes } from "./utils";
|
||||
import { TablesManager } from "./dataLoader/collections/tablesManager";
|
||||
import { sanitise } from "./utils/sanitiseColumn";
|
||||
import { AFileSyncTable } from "./fileSyncTable/abstractFileSyncTable";
|
||||
|
||||
function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<string, any>) {
|
||||
return {
|
||||
...frontmatter,
|
||||
path: file.path,
|
||||
name: file.name.replace(/\.[^/.]+$/, ""),
|
||||
id: file.path
|
||||
}
|
||||
}
|
||||
import { App, Plugin, TFile } from "obsidian";
|
||||
import { FieldTypes } from "src/utils/typePredictions";
|
||||
import { sanitise } from "src/utils/sanitiseColumn";
|
||||
import { AFileSyncTable } from "./tables/abstractFileSyncTable";
|
||||
|
||||
const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
|
||||
const frontmatter = plugin.app.metadataCache.getFileCache(file)?.frontmatter || {}
|
||||
|
|
@ -1,12 +1,10 @@
|
|||
import { App, TFile } from "obsidian";
|
||||
import { SqlSealDatabase } from "src/database";
|
||||
import { TablesManager } from "src/dataLoader/collections/tablesManager";
|
||||
import { SqlSealDatabase } from "src/database/database";
|
||||
|
||||
export abstract class AFileSyncTable {
|
||||
constructor(
|
||||
protected readonly db: SqlSealDatabase,
|
||||
protected readonly app: App,
|
||||
protected tableManager: TablesManager
|
||||
protected readonly app: App
|
||||
) {
|
||||
}
|
||||
|
||||
|
|
@ -1,9 +1,8 @@
|
|||
import { App, getAllTags, Plugin, TAbstractFile, TFile } from "obsidian";
|
||||
import { App, Plugin, TAbstractFile, TFile } from "obsidian";
|
||||
import { AFileSyncTable } from "./abstractFileSyncTable";
|
||||
import { sanitise } from "src/utils/sanitiseColumn";
|
||||
import { SqlSealDatabase } from "src/database";
|
||||
import { TablesManager } from "src/dataLoader/collections/tablesManager";
|
||||
import { predictType } from "src/utils";
|
||||
import { SqlSealDatabase } from "src/database/database";
|
||||
import { predictType } from "src/utils/typePredictions";
|
||||
|
||||
|
||||
const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
|
||||
|
|
@ -27,19 +26,17 @@ function fileData(file: TAbstractFile, {tags: _tags, ...frontmatter }: Record<st
|
|||
export class FilesFileSyncTable extends AFileSyncTable {
|
||||
private currentSchema: Record<string, ReturnType<typeof predictType>>
|
||||
shouldPerformBulkInsert = true;
|
||||
constructor(db: SqlSealDatabase, app: App, tableManager: TablesManager, private readonly plugin: Plugin) {
|
||||
super(db, app, tableManager)
|
||||
constructor(db: SqlSealDatabase, app: App, private readonly plugin: Plugin) {
|
||||
super(db, app)
|
||||
}
|
||||
async onFileModify(file: TFile): Promise<void> {
|
||||
// we need to update the row
|
||||
const frontmatter = await extractFrontmatterFromFile(file, this.plugin)
|
||||
await this.db.updateData('files', [fileData(file, frontmatter)])
|
||||
this.tableManager.getTableSignal('files')(Date.now())
|
||||
await sleep(1000) // TO DELAY OTHER PLUGINS, UGLY BUT WORKS.
|
||||
}
|
||||
async onFileDelete(path: string): Promise<void> {
|
||||
await this.db.deleteData('files', [{ id: path }])
|
||||
this.tableManager.getTableSignal('files')(Date.now())
|
||||
}
|
||||
|
||||
async onFileCreate(file: TFile): Promise<void> {
|
||||
|
|
@ -49,7 +46,6 @@ export class FilesFileSyncTable extends AFileSyncTable {
|
|||
// TODO: Check if there are new columns. If there are, we update the database.
|
||||
|
||||
await this.db.insertData('files', [fileData(file, frontmatter)])
|
||||
this.tableManager.getTableSignal('files')(Date.now())
|
||||
|
||||
// FIXME: should we be waiting here?
|
||||
await sleep(1000)
|
||||
|
|
@ -60,7 +56,6 @@ export class FilesFileSyncTable extends AFileSyncTable {
|
|||
const data = await Promise.all(files.map(async f => fileData(f, await extractFrontmatterFromFile(f, this.plugin))))
|
||||
const schema = await this.db.createTableWithData('files', data)
|
||||
this.currentSchema = schema
|
||||
this.tableManager.getTableSignal('files')(Date.now())
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -8,13 +8,11 @@ export class TagsFileSyncTable extends AFileSyncTable {
|
|||
}
|
||||
async onFileDelete(path: string): Promise<void> {
|
||||
await this.db.deleteData('tags', [{ fileId: path }], 'fileId')
|
||||
this.tableManager.getTableSignal('tags')(Date.now())
|
||||
}
|
||||
|
||||
async onFileCreate(file: TFile): Promise<void> {
|
||||
const tags = await this.getFileTags(file)
|
||||
this.db.insertData('tags', tags)
|
||||
this.tableManager.getTableSignal('tags')(Date.now())
|
||||
}
|
||||
|
||||
private async getFileTags(file: TFile) {
|
||||
|
|
@ -38,6 +36,5 @@ export class TagsFileSyncTable extends AFileSyncTable {
|
|||
'tag': 'TEXT',
|
||||
'fileId': 'TEXT'
|
||||
})
|
||||
this.tableManager.getTableSignal('tags')(Date.now())
|
||||
}
|
||||
}
|
||||
|
|
@ -8,13 +8,11 @@ export class TasksFileSyncTable extends AFileSyncTable {
|
|||
}
|
||||
async onFileDelete(path: string): Promise<void> {
|
||||
await this.db.deleteData('tasks', [{ filePath: path }], 'filePath')
|
||||
this.tableManager.getTableSignal('tasks')(Date.now())
|
||||
}
|
||||
|
||||
async onFileCreate(file: TFile): Promise<void> {
|
||||
const tasks = await this.getFileTags(file)
|
||||
this.db.insertData('tasks', tasks)
|
||||
this.tableManager.getTableSignal('tasks')(Date.now())
|
||||
}
|
||||
|
||||
async getFileTags(file: TFile) {
|
||||
|
|
@ -52,6 +50,5 @@ export class TasksFileSyncTable extends AFileSyncTable {
|
|||
'completed': 'INTEGER',
|
||||
'filePath': 'TEXT'
|
||||
})
|
||||
this.tableManager.getTableSignal('tasks')(Date.now())
|
||||
}
|
||||
}
|
||||
|
|
@ -18,5 +18,6 @@
|
|||
"0.12.1": "0.15.0",
|
||||
"0.12.2": "0.15.0",
|
||||
"0.12.3": "0.15.0",
|
||||
"0.12.4": "0.15.0"
|
||||
"0.12.4": "0.15.0",
|
||||
"0.13.0": "0.15.0"
|
||||
}
|
||||
Loading…
Reference in a new issue