obsidian-plugin/h-sphere_sql-seal
1
0
Fork 0
mirror of https://github.com/h-sphere/sql-seal.git synced 2026-07-22 10:10:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #7 from h-sphere/fix/csv-fix

Browse source 
[0.6.0] Many small fixes
This commit is contained in:
Kacper Kula 2024-09-01 20:01:10 +01:00 committed by GitHub
commit 645c74daa3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 263 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
CHANGELOG.md
View file

@ -1,3 +1,9 @@
# 0.6.0
- fixing CSV loading when names has been converted to camelCase
- Fixing constantly reloading files when modifying sql inside the same block that has table declaration in
- Auto-parsing JSON values with JSON5
- Better errors when error occurs in select statement
# 0.5.0
- Adding support for JSON objects! Automatically detecting JSON in the frontmatter and converting it to JSON type in SQLite. You can query fields using built-in SQLite functions like `json_extract`, `json_array_length`, etc. More about SQlite functionality [can be found here.](https://www.sqlite.org/json1.html).
- Fixed issue with bookean types not being saved properly.

13
main.ts
View file

@ -12,17 +12,15 @@ export default class SqlSealPlugin extends Plugin {
async onload() {
await this.loadSettings();
const sqlSeal = new SqlSeal(this.app, false)
const sqlSeal = new SqlSeal(this.app, false) // FIXME: set verbose based on the env.
this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler())
await sqlSeal.connect()
this.sqlSeal = sqlSeal
this.fileSync = new SealFileSync(this.app, sqlSeal, this)
// start syncing when files are loaded
this.app.workspace.onLayoutReady(() => {
sqlSeal.db.connect().then(() => {
this.fileSync = new SealFileSync(this.app, sqlSeal, this)
this.fileSync.init()
})
})
// this.addSettingTab(new SqlSealSettingsTab(this.app, this));
@ -30,9 +28,6 @@ export default class SqlSealPlugin extends Plugin {
}
onunload() {
if (this.fileSync) {
this.fileSync.destroy();
}
this.sqlSeal.db.disconect();
}

2
manifest.json
View file

@ -1,7 +1,7 @@
{
"id": "sqlseal",
"name": "SQLSeal",
"version": "0.5.0",
"version": "0.6.0",
"minAppVersion": "0.15.0",
"description": "Use SQL in your notes to query your vault files and CSV content.",
"author": "hypersphere",

3
package.json
View file

@ -1,6 +1,6 @@
{
"name": "sqlseal",
"version": "0.5.0",
"version": "0.6.0",
"description": "A plugin for Obsidian that allows you to run SQL queries on your notes.",
"main": "main.js",
"scripts": {
@ -43,6 +43,7 @@
},
"dependencies": {
"better-sqlite3": "^11.2.1",
"json5": "^2.2.3",
"lodash": "^4.17.21",
"node-sql-parser": "^5.0.0",
"papaparse": "^5.4.1"

3
pnpm-lock.yaml
View file

@ -11,6 +11,9 @@ importers:
better-sqlite3:
specifier: ^11.2.1
version: 11.2.1
json5:
specifier: ^2.2.3
version: 2.2.3
lodash:
specifier: ^4.17.21
version: 4.17.21

3
src/SealFileSync.ts
View file

@ -1,5 +1,6 @@
import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian";
import { SqlSeal } from "./sqlSeal";
import { FieldTypes } from "./utils";
function fileData(file: TAbstractFile, frontmatter: Record<string, any>) {
return {
@ -15,7 +16,7 @@ const extractFrontmatterFromFile = async (file: TFile, plugin: Plugin) => {
}
export class SealFileSync {
private currentSchema: Record<string, 'TEXT' | 'INTEGER' | 'REAL'> = {}
private currentSchema: Record<string, FieldTypes> = {}
constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) {
plugin.registerEvent(this.app.vault.on('modify', async (file) => {
if (!(file instanceof TFile)) {

22
src/SealObserver.ts
View file

@ -1,15 +1,17 @@
import { Logger } from "./logger";
type Callback = () => void;
export class SealObserver {
private tables: Map<string, Set<Callback>>;
private verbose: boolean;
private tags: Map<string, Set<Callback>>;
private logger: Logger
constructor(verbose = false) {
this.tables = new Map();
this.tags = new Map();
this.verbose = verbose;
this.logger = new Logger(verbose)
}
registerObserver(tableNames: string | string[], observer: Callback, tag?: string) {
@ -19,16 +21,12 @@ export class SealObserver {
tableNames.forEach(tableName => {
if (!this.tables.has(tableName)) {
this.tables.set(tableName, new Set<Callback>());
if (this.verbose) {
console.log(`Table "${tableName}" registered.`);
}
this.logger.log(`Table "${tableName}" registered.`);
}
const observers = this.tables.get(tableName);
if (observers !== undefined) {
observers.add(observer);
if (this.verbose) {
console.log(`Observer registered for table "${tableName}".`);
}
this.logger.log(`Observer registered for "${tableName}".`);
}
if (tag) {
if (!this.tags.has(tag)) {
@ -58,9 +56,7 @@ export class SealObserver {
this.tables.forEach(observers => {
observers.delete(observer);
});
if (this.verbose) {
console.log(`Observer unregistered.`);
}
this.logger.log(`Observer unregistered.`);
}
fireObservers(tableName: string) {
@ -68,9 +64,7 @@ export class SealObserver {
const observers = this.tables.get(tableName);
if (observers) {
observers.forEach(observer => observer());
if (this.verbose) {
console.log(`Observers fired for table "${tableName}".`);
}
this.logger.log(`Observers fired for table "${tableName}".`);
}
}
}

113
src/SqlSealCodeblockHandler.ts Normal file
View file

@ -0,0 +1,113 @@
import { App, MarkdownPostProcessorContext } from "obsidian"
import { displayData, displayError, displayInfo, displayLoader } from "./ui"
import { resolveFrontmatter } from "./frontmatter"
import { hashString } from "./hash"
import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables"
import { SealObserver } from "./SealObserver"
import { SqlSealDatabase } from "./database"
import { Logger } from "./logger"
export class SqlSealCodeblockHandler {
get globalTables() {
return ['files', 'tags']
}
constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) { }
tablesConfig: Record<string, string> = {}
getHandler() {
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
displayLoader(el)
await this.db.connect()
this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers.
const frontmatter = await resolveFrontmatter(ctx, this.app)
const prefix = hashString(ctx.sourcePath)
const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g;
let match
while ((match = regex.exec(source)) !== null) {
const name = match[1];
const url = match[2];
this.logger.log('CodedblockHandler. table', name, url)
// UPDATING TABLES IF THEY CHANGED SINCE LAST REGISTER
const prefixedName = prefixedIfNotGlobal(name, this.globalTables, prefix)
if (this.tablesConfig[prefixedName] === url) {
// We do not need to rework it. Also, it should be watched, right?
// continue
}
if (!!this.tablesConfig[prefixedName]) {
// We need to remove old database as it's pointing to the wrong collection (probably).
}
// FIXME: do not register observer when it's already been registed for this ctx.
this.observer.registerObserver(`file:${url}`, async () => {
// Update table
await this.db.loadDataForDatabaseFromUrl(prefixedName, url, true)
// Fire observers for the table
this.observer.fireObservers(`table:${prefixedName}`)
}, ctx.docId)
if (this.tablesConfig[prefixedName] !== url) {
this.observer.fireObservers(`file:${url}`)
this.tablesConfig[prefixedName] = url
} else {
// We still need to update the table watchers?
requestAnimationFrame(() => {
this.observer.fireObservers(`table:${prefixedName}`)
})
}
}
const selectRegexp = /SELECT\s+(.*)/g; // OR WITH?
const selectMatch = selectRegexp.exec(source)
if (selectMatch) {
try {
// FIXME: WAIT FOR TABLES TO BE READDY!!!!
const selectStatement = selectMatch[0]
const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix)
const renderSelect = async () => {
try {
const stmt = await this.db.db.prepare(statement)
const columns = await stmt.columns().map(column => column.name);
const data = await stmt.all(frontmatter ?? {})
displayData(el, columns, data)
} catch (e) {
displayError(el, e)
}
}
// Register observer for each table
tables.forEach(table => {
const observer = async () => {
if (!el.isConnected) {
// Unregistering using the context id
this.observer.unregisterObserversByTag(ctx.docId)
}
displayLoader(el)
await renderSelect()
}
this.observer.registerObserver(`table:${table}`, observer, ctx.docId)
})
this.globalTables.forEach(t => this.observer.fireObservers(`table:${t}`))
// await renderSelect() <- this should get triggered automatically.
} catch (e) {
if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) {
displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.')
} else {
displayError(el, e)
}
}
}
}
}
}

7
src/csvParse.ts
View file

@ -1,7 +0,0 @@
interface Results {
columns: Record<string, string>,
data: Record<string, string>[]
}
const parseCsv = (file) => {
}

81
src/database.ts
View file

@ -4,20 +4,20 @@ import path from 'path'
import Papa from 'papaparse'
import { prefixedIfNotGlobal } from "./sqlReparseTables"
import { camelCase } from 'lodash'
import { fetchBlobData, predictType } from "./utils"
import { fetchBlobData, FieldTypes, predictJson, predictType } from "./utils"
import os from 'os'
import fs from 'fs'
export function isNumeric(str: string) {
if (typeof str != "string") return false // we only process strings!
return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
function dataToCamelCase(data: Array<Record<string, unknown>>) {
return data.map(d => Object.keys(d).reduce((acc, k) => ({
...acc,
[camelCase(k)]: Array.isArray(d[k]) ? d[k].join(', ') : d[k]
[camelCase(k)]: d[k]
}), {}))
}
@ -48,10 +48,10 @@ const toTypeStatements = (header: Array<string>, data: Array<Record<string, stri
export class SqlSealDatabase {
private savedDatabases: Record<string, any> = {}
db: typeof Database
db: Database.Database
private isConnected = false
private connectingPromise;
private connectingPromiseResolve;
private connectingPromise: Promise<void>;
private connectingPromiseResolve: (value: void | PromiseLike<void>) => void
constructor(private readonly app: App, private readonly verbose = false) {
}
@ -132,7 +132,8 @@ export class SqlSealDatabase {
return
}
const alter = this.db.prepare(`ALTER TABLE ${name} ADD COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`)
const alter = this.db.prepare(`ALTER TABLE ${name} ADD
COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`)
alter.run()
}
@ -168,7 +169,7 @@ export class SqlSealDatabase {
return deleteMany(data)
}
insertData(name: string, inData: Array<Record<string, unknown>>) {
async insertData(name: string, inData: Array<Record<string, unknown>>) {
const data = dataToCamelCase(inData)
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
// FIXME: reworking all fields to be camel case
@ -185,7 +186,7 @@ export class SqlSealDatabase {
data[field] = data[field] ? 1 : 0
} else if (!data[field]) {
data[field] = null
} else if (typeof data[field] === 'object') {
} else if (typeof data[field] === 'object' || Array.isArray(data[field])) {
data[field] = JSON.stringify(data[field])
}
})
@ -196,25 +197,25 @@ export class SqlSealDatabase {
})
})
return insertMany(data)
insertMany(data)
}
async createTable(name: string, fields: Record<string, 'TEXT' | 'INTEGER' | 'REAL'>) {
async createTable(name: string, fields: Record<string, FieldTypes>) {
const transformedFiels = Object.entries(fields).map(([key, type]) => [camelCase(key), type])
const uniqueFields = [...new Map(transformedFiels.map(item =>
[item[0], item])).values()]
const sqlFields = uniqueFields.map(([key, type]) => `${key} ${type}`)
// FIXME: probably use schema generator, for now create with hardcoded fields
await this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
const createSQL = `CREATE TABLE IF NOT EXISTS ${name} (
${sqlFields.join(', ')}
);`
await this.db.prepare(createSQL).run()
this.db.prepare(createSQL).run()
this.savedDatabases[name] = true
// Dropping data.
await this.db.prepare(`DELETE FROM ${name}`).run()
this.db.prepare(`DELETE FROM ${name}`).run()
}
async getSchema(data: Array<Record<string, unknown>>) {
const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
@ -222,36 +223,47 @@ export class SqlSealDatabase {
return types;
}
async loadDataForDatabaseFromUrl(name: string, url: string) {
async loadDataForDatabaseFromUrl(name: string, url: string, reloadData: boolean = false) {
const file = this.app.vault.getFileByPath(url)
if (!file) {
return
}
const data = await this.app.vault.cachedRead(file)
const parsed = Papa.parse(data, {
const parsed = Papa.parse<Record<string, string>>(data, {
header: true,
dynamicTyping: false,
skipEmptyLines: true
})
const fields = parsed.meta.fields
const { data: parsedData, types } = toTypeStatements(fields, parsed.data)
// Purge the database
await this.db.prepare(`DELETE FROM ${name}`).run()
const processedData = dataToCamelCase(parsed.data)
const processedWithJsonParsed = predictJson(processedData)
const fields = parsed.meta.fields?.map((f: string) => camelCase(f))!
const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`);
const insertMany = this.db.transaction((pData: Array<Record<string, any>>) => {
pData.forEach(data => {
try {
insert.run(data)
} catch (e) {
console.error(e)
}
})
})
const { data: parsedData, types } = toTypeStatements(fields, processedWithJsonParsed)
await insertMany(parsedData)
try {
// Purge the database
await this.db.prepare(`DELETE FROM ${name}`).run()
} catch (e) {
// FIXME: check if error is actually that the table does not exist
await this.createTable(name, types)
}
await this.insertData(name, parsedData)
return name
}
/**
* @deprecated
* @param unprefixedName
* @param url
* @param prefix
* @param reloadData
* @returns
*/
async defineDatabaseFromUrl(unprefixedName: string, url: string, prefix: string, reloadData: boolean = false) {
// FIXME: why do we repeat this code?
const name = prefixedIfNotGlobal(unprefixedName, [], prefix) // FIXME: should we pass global tables here too?
if (this.savedDatabases[name]) {
if (reloadData) {
@ -260,21 +272,22 @@ export class SqlSealDatabase {
return name
}
const file = this.app.vault.getFileByPath(url)
if (!file) {
return name
}
const data = await this.app.vault.cachedRead(file)
const parsed = Papa.parse(data, {
const parsed = Papa.parse<Record<string, string>>(data, {
header: true,
dynamicTyping: false,
skipEmptyLines: true
})
const fields = parsed.meta.fields
const fields = parsed.meta.fields!
const { data: parsedData, types } = toTypeStatements(fields, parsed.data)
await this.createTable(name, types)
// this.savedDatabases[name] = url
this.savedDatabases[name] = url
await this.loadDataForDatabaseFromUrl(name, url)

12
src/frontmatter.ts Normal file
View file

@ -0,0 +1,12 @@
import { App, MarkdownPostProcessorContext } from "obsidian"
export const resolveFrontmatter = (ctx: Pick<MarkdownPostProcessorContext, 'frontmatter' | 'sourcePath'>, app: App) => {
if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) {
return ctx.frontmatter as Record<string, any>
}
const file = app.vault.getFileByPath(ctx.sourcePath)
if (!file) {
return null
}
return app.metadataCache.getFileCache(file)?.frontmatter
}

22
src/logger.ts Normal file
View file

@ -0,0 +1,22 @@
export class Logger {
constructor(verbose: boolean) {
if (verbose) {
this.console = console
} else {
this.console = {
log: () => { },
error: () => { }
}
}
}
private console: Pick<typeof console, 'log' | 'error'>
log(...args: any[]) {
this.console.log(...args)
}
error(...args: any[]) {
this.console.error(...args)
}
}

101
src/sqlSeal.ts
View file

@ -1,113 +1,30 @@
import { SqlSealDatabase } from "./database";
import { displayData, displayError, displayInfo, displayLoader } from "./ui";
import { App, MarkdownPostProcessorContext } from "obsidian";
import { updateTables } from "./sqlReparseTables";
import { hashString } from "./hash";
import { App } from "obsidian";
import { SealObserver } from "./SealObserver";
import { SqlSealCodeblockHandler } from "./SqlSealCodeblockHandler";
import { Logger } from "./logger";
export class SqlSeal {
public db: SqlSealDatabase
public observer: SealObserver
public codeBlockHandler: SqlSealCodeblockHandler
constructor(private readonly app: App, verbose = false) {
this.db = new SqlSealDatabase(app, verbose)
this.observer = new SealObserver(verbose)
this.observeAllFileChanges()
const logger = new Logger(verbose)
this.codeBlockHandler = new SqlSealCodeblockHandler(app, this.db, this.observer, logger)
// FIXME: handle here changes to files and tags?
}
public get globalTables() {
return ['files', 'tags']
getHandler() {
return this.codeBlockHandler.getHandler()
}
async connect() {
await this.db.connect()
}
async disconnect() {
await this.db.disconect()
}
async resolveFrontmatter(ctx: MarkdownPostProcessorContext) {
if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) {
return ctx.frontmatter as Record<string, any>
}
const file = this.app.vault.getFileByPath(ctx.sourcePath)
if (!file) {
return null
}
return this.app.metadataCache.getFileCache(file)?.frontmatter
}
private observeAllFileChanges() {
// Use fs to observe file changes
this.app.vault.on('modify', async (file) => {
this.observer.fireObservers('file:' + file.path)
})
}
getHandler() {
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
displayLoader(el)
await this.connect()
const frontmatter = await this.resolveFrontmatter(ctx)
const prefix = hashString(ctx.sourcePath)
const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g;
let match
while ((match = regex.exec(source)) !== null) {
const name = match[1];
const url = match[2];
const prefixedName = await this.db.defineDatabaseFromUrl(name, url, prefix)
this.observer.registerObserver(`file:${url}`, async () => {
// Update table
await this.db.defineDatabaseFromUrl(name, url, prefix, true)
// Fire observers for the table
this.observer.fireObservers(`table:${prefixedName}`)
}, ctx.docId)
}
const selectRegexp = /SELECT\s+(.*)/g;
const selectMatch = selectRegexp.exec(source)
if (selectMatch) {
try {
const selectStatement = selectMatch[0]
const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix)
const renderSelect = async () => {
const stmt = await this.db.db.prepare(statement)
const columns = await stmt.columns().map(column => column.name);
const data = await stmt.all(frontmatter ?? {})
displayData(el, columns, data)
}
// Register observer for each table
tables.forEach(table => {
const observer = async () => {
if (!el.isConnected) {
// Unregistering using the context id
this.observer.unregisterObserversByTag(ctx.docId)
}
displayLoader(el)
await renderSelect()
}
this.observer.registerObserver(`table:${table}`, observer, ctx.docId)
})
await renderSelect()
} catch (e) {
if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) {
displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.')
} else {
displayError(el, e)
}
}
}
}
}
}

30
src/utils.ts
View file

@ -1,5 +1,6 @@
import { get } from "https"
import JSON5 from 'json5'
import * as fs from 'fs'
import { isNumeric } from "./database";
@ -77,3 +78,32 @@ export const predictType = (field: string, data: Array<Record<string, string | O
}
return 'TEXT'
}
export type FieldTypes = ReturnType<typeof predictType>
export const predictJson = (data: Array<Record<string, unknown>>) => {
return data.map(d => Object.keys(d).reduce((o, k) => {
if (typeof d[k] !== 'string') {
return {
...o,
[k]: d[k]
}
}
const v = d[k].trim()
if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) {
try {
const d = JSON5.parse(v)
return {
...o,
[k]: d
}
} catch (e) {
}
}
return {
...o,
[k]: d[k]
}
}, {}))
}

3
tsconfig.json
View file

@ -18,7 +18,8 @@
"ES5",
"ES6",
"ES7"
]
],
"allowSyntheticDefaultImports": true
},
"include": [
"**/*.ts"

3
versions.json
View file

@ -4,5 +4,6 @@
"0.3.0": "0.15.0",
"0.4.0": "0.15.0",
"0.4.1": "0.15.0",
"0.5.0": "0.15.0"
"0.5.0": "0.15.0",
"0.6.0": "0.15.0"
}