diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4dbcf58 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# 0.2.0 +- Added ability to query files in the fault directly. +- Added observability - when CSV or file in the vault is changed, all SELECTS that uses it should update too +- Added custom class to sqlseal tables and ability to scroll vertically when the data is overflowing horizontally. + +# 0.1.0 +- Initial release. Allows to create tables based on CSV files in your vault and query them using SQL. \ No newline at end of file diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index dad87f5..7d6c6b9 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -18,8 +18,8 @@ export default defineConfig({ items: [ { text: 'Quick Start', link: '/quick-start' }, { text: 'Using properties', link: '/using-properties' }, - { text: 'Future Plans', link: '/future-plans' } - + { text: 'Future Plans', link: '/future-plans' }, + { text: 'Query Vault Content', link: '/query-vault-content' }, ] } ], @@ -29,7 +29,7 @@ export default defineConfig({ ], footer: { message: '', - copyright: 'By hypersphere' + copyright: 'By hypersphere. Ko-Fi: hypersphere' } } }) diff --git a/docs/index.md b/docs/index.md index 8317466..0588c9d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,7 +16,7 @@ features: details: Use full power of SQL to select, join, filter data for your liking - title: Fully featured SQL Engine details: With SQLite under the hood, you can use all functionality of the database - - title: Zero configuration - details: Just use SQL syntax to create and query the data + - title: Query your files and tags + details: Use SQL to filter files in your vault --- diff --git a/docs/query-vault-content.md b/docs/query-vault-content.md new file mode 100644 index 0000000..cc416ac --- /dev/null +++ b/docs/query-vault-content.md @@ -0,0 +1,27 @@ +# Query your Vault content + +You can query your vault content using SQLSeal built-in tables `files` and `tags`. You can use them to query specific files in the fault based on Properties (Frontmatter) and associated tags. + +## Example: Get all files from the vault +To get all files from the fault you can run the following query: + +```sqlseal +SELECT * FROM files +``` + +## Filter by Properties +If your files have frontmatter properties, you can query by them using SQL `WHERE` clause. SQLSeal automatically maintains SQL schema and creates columns when needed. Let's assume we have files with property `type`. We can query only specific notes by running the following: + +```sqlseal +SELECT * FROM files WHERE type = 'resource' +``` + +The query above will return only files that have property `type` set to value `resource`. + +## Filter by Tags +Tags are kept in a separate table `tags`. To select all files that have specific tag, we can perform simple join. + +```sqlseal +SELECT files.* FROM files JOIN tags ON files.id=tags.fileId WHERE tag = '#important' +``` + diff --git a/main.ts b/main.ts index fc9ff37..88f737b 100644 --- a/main.ts +++ b/main.ts @@ -1,4 +1,5 @@ import { Plugin } from 'obsidian'; +import { SealFileSync } from 'src/SealFileSync'; import { SqlSealSettings } from 'src/settings'; import { SqlSeal } from 'src/sqlSeal'; @@ -6,28 +7,33 @@ const DEFAULT_SETTINGS = { rows: [] } export default class SqlSealPlugin extends Plugin { settings: SqlSealSettings; + fileSync: SealFileSync; + sqlSeal: SqlSeal; async onload() { await this.loadSettings(); const sqlSeal = new SqlSeal(this.app, false) this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) await sqlSeal.connect() + + this.sqlSeal = sqlSeal - // const fileSync = new SealFileSync(this.app, sqlSeal) - // setTimeout(() => { - // fileSync.init() - // }, 5000) + this.fileSync = new SealFileSync(this.app, sqlSeal, this) + // start syncing when files are loaded + this.app.workspace.onLayoutReady(() => { + this.fileSync.init() + }) // this.addSettingTab(new SqlSealSettingsTab(this.app, this)); - - - } onunload() { - + if (this.fileSync) { + this.fileSync.destroy(); + } + this.sqlSeal.db.disconect(); } async loadSettings() { diff --git a/manifest.json b/manifest.json index 3ea9832..4bcd972 100644 --- a/manifest.json +++ b/manifest.json @@ -1,11 +1,11 @@ { "id": "hypersphere-sqlseal", "name": "SQL Seal", - "version": "0.1.0", + "version": "0.2.0", "minAppVersion": "0.15.0", - "description": "Use SQL in your notes!", + "description": "Use SQL in your notes to query your vault and CSV files.", "author": "hypersphere", "authorUrl": "https://hypersphere.blog/", - "fundingUrl": "https://hypersphere.blog", + "fundingUrl": "ko-fi.com/hypersphere", "isDesktopOnly": false } diff --git a/package.json b/package.json index 314de83..e8c2bc0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "obsidian-sqlseal", - "version": "0.1.0", + "version": "0.2.0", "description": "A plugin for Obsidian that allows you to run SQL queries on your notes.", "main": "main.js", "scripts": { diff --git a/src/SealFileSync.ts b/src/SealFileSync.ts index 89b3a06..f6ad495 100644 --- a/src/SealFileSync.ts +++ b/src/SealFileSync.ts @@ -1,12 +1,14 @@ -import { App, TAbstractFile } from "obsidian"; +import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian"; import GrayMatter from "gray-matter"; import { SqlSeal } from "./sqlSeal"; +import { delay } from "./utils"; async function extractFrontmatterFromFile(file: TAbstractFile) { const content = await this.app.vault.read(file); const gm = GrayMatter(content) + return gm.data } @@ -21,36 +23,144 @@ function fileData(file: TAbstractFile, frontmatter: Record) { export class SealFileSync { private currentSchema: Record = {} - constructor(public readonly app: App, private readonly sqlSeal: SqlSeal) { - this.app.vault.on('modify', async (file) => { - console.log('Modified', file) - const frontmatter = await extractFrontmatterFromFile(file) - if (this.isFrontmatterSame([frontmatter])) { - // we need to update the row - await this.sqlSeal.db.updateData('files', [fileData(file, frontmatter)]) + private refs: Array = [] + constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) { + this.refs.push(this.app.vault.on('modify', async (file) => { + if (!(file instanceof TFile)) { return } - await this.init() + console.log('Modified', file) + const frontmatter = await extractFrontmatterFromFile(file) + + if (this.hasNewColumns(frontmatter)) { + await delay(1000) + await this.init() + + return + } + // we need to update the row + await this.sqlSeal.db.updateData('files', [fileData(file, frontmatter)]) + await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId') + + this.sqlSeal.observer.fireObservers('table:files') + + + // Wait 1 second before updating tags table + await delay(1000) + await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) + this.sqlSeal.observer.fireObservers('table:tags') + })) + + this.refs.push(this.app.vault.on('create', async (file) => { + if (!(file instanceof TFile)) { + return + } + console.log('File created', file.path) + const frontmatter = await extractFrontmatterFromFile(file) + + + if (this.hasNewColumns(frontmatter)) { + await delay(1000) + await this.init() + + return + } + + + // we need to update the row + await this.sqlSeal.db.insertData('files', [fileData(file, frontmatter)]) + this.sqlSeal.observer.fireObservers('table:files') + + // Wait 1 second before updating tags table + await delay(1000) + await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) + this.sqlSeal.observer.fireObservers('table:tags') + })) + + this.refs.push(this.app.vault.on('delete', async (file) => { + if (!(file instanceof TFile)) { + return + } + + console.log('File deleted', file.path) + await this.sqlSeal.db.deleteData('files', [{ id: file.path }]) + this.sqlSeal.observer.fireObservers('table:files') + + await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId') + this.sqlSeal.observer.fireObservers('table:tags') + })) + + this.refs.push(this.app.vault.on('rename', async (file, oldPath) => { + if (!(file instanceof TFile)) { + return + } + + console.log('File renamed', file.path) + // deleting old one and adding new one + await this.sqlSeal.db.deleteData('files', [{ id: oldPath }]) + + // delete old tags + await this.sqlSeal.db.deleteData('tags', [{ fileId: oldPath }], 'fileId') + + await this.sqlSeal.db.insertData('files', [fileData(file, await extractFrontmatterFromFile(file))]) + this.sqlSeal.observer.fireObservers('table:files') + + // Wait 1 second before updating tags table + await delay(1000) + await this.sqlSeal.db.insertData('tags', await this.getFileTags(file)) + this.sqlSeal.observer.fireObservers('table:tags') + + })) + + // add Obsidian command to reload SQLSeal file database + this.plugin.addCommand({ + id: 'reload-sqlseal', + name: 'Reload Vault Database', + callback: async () => { + await this.init() + } }) } + + destroy() { + this.refs.forEach(ref => this.app.vault.offref(ref)) + } + + async getFileTags(file: TFile) { + + // Get fresh tag + console.log('FILE', file) + + + return (this.app.metadataCache.getFileCache(file)?.tags || []) + .map(t => t.tag) + .map(t => ({ tag: t, fileId: file.path })) + } + async init() { const files = this.app.vault.getMarkdownFiles(); const data = [] + const tags: Array<{fileId: string, tag: string }> = [] for (const file of files) { const frontmatter = await extractFrontmatterFromFile(file) + tags.push(...await this.getFileTags(file)) data.push(fileData(file, frontmatter)) } - const schema = await this.sqlSeal.db.createTableWithData('files', data) this.currentSchema = schema + await this.sqlSeal.db.createTableWithData('tags', tags) + this.sqlSeal.observer.fireObservers('table:files') + this.sqlSeal.observer.fireObservers('table:tags') } - isFrontmatterSame(newFrontmatter: Array>) { - const newSchema = this.sqlSeal.db.getSchema(newFrontmatter) - return JSON.stringify(newSchema) === JSON.stringify(this.currentSchema) + hasNewColumns(newFrontmatter: Record) { + const currentFields = Object.keys(this.currentSchema) + const newFields = Object.keys(newFrontmatter).filter(f => !currentFields.includes(f)) + console.log('current fields', currentFields, Object.keys(newFrontmatter)) + return newFields.length > 0 } } \ No newline at end of file diff --git a/src/database.ts b/src/database.ts index a62048b..0d8f7fa 100644 --- a/src/database.ts +++ b/src/database.ts @@ -105,6 +105,14 @@ export class SqlSealDatabase { this.connectingPromiseResolve() } + async disconect() { + if (!this.isConnected) { + return + } + this.db.close() + this.isConnected = false + } + async createTableWithData(name: string, data: Array>) { const schema = await this.getSchema(data) await this.createTable(name, schema) @@ -113,6 +121,20 @@ export class SqlSealDatabase { return schema } + async addNewColumns(name: string, data: Array>) { + const schema = await this.getSchema(data) + const currentSchema = this.db.prepare(`PRAGMA table_info(${name})`).all() + const currentFields = currentSchema.map((f: any) => f.name) + const newFields = Object.keys(schema).filter(f => !currentFields.includes(f)) + + if (newFields.length === 0) { + return + } + + const alter = this.db.prepare(`ALTER TABLE ${name} ADD COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`) + alter.run() + } + updateData(name: string, data: Array>) { const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); const update = this.db.prepare(`UPDATE ${name} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`); @@ -129,6 +151,22 @@ export class SqlSealDatabase { return updateMany(data) } + deleteData(name: string, data: Array>, key: string = 'id') { + + const deleteStmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`); + const deleteMany = this.db.transaction((pData: Array>) => { + pData.forEach(data => { + try { + deleteStmt.run(data) + } catch (e) { + console.log(e) + } + }) + }) + + return deleteMany(data) + } + insertData(name: string, data: Array>) { const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {})); const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); @@ -201,7 +239,7 @@ export class SqlSealDatabase { } async defineDatabaseFromUrl(unprefixedName: string, url: string, prefix: string, reloadData: boolean = false) { - const name = prefixedIfNotGlobal(unprefixedName, [], prefix) + const name = prefixedIfNotGlobal(unprefixedName, [], prefix) // FIXME: should we pass global tables here too? if (this.savedDatabases[name]) { console.log('Database Exists', name) if (reloadData) { diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index e5b5eda..09749b2 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -9,17 +9,25 @@ import { delay } from "./utils"; export class SqlSeal { public db: SqlSealDatabase - private observer: SealObserver + public observer: SealObserver constructor(private readonly app: App, verbose = false) { this.db = new SqlSealDatabase(app, verbose) this.observer = new SealObserver(verbose) this.observeAllFileChanges() } + public get globalTables() { + return ['files', 'tags'] + } + async connect() { await this.db.connect() } + async disconnect() { + await this.db.disconect() + } + async resolveFrontmatter(ctx: MarkdownPostProcessorContext) { if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) { return ctx.frontmatter as Record @@ -73,7 +81,7 @@ export class SqlSeal { if (selectMatch) { try { const selectStatement = selectMatch[0] - const { statement, tables } = updateTables(selectStatement, ['files'], prefix) + const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix) const renderSelect = async () => { diff --git a/src/ui.ts b/src/ui.ts index f7296ba..156bcd0 100644 --- a/src/ui.ts +++ b/src/ui.ts @@ -1,6 +1,9 @@ export const displayData = (el: HTMLElement, columns, data) => { el.empty() - const table = el.createEl("table") + const container = el.createDiv({ + cls: 'sqlseal-table-container' + }) + const table = container.createEl("table") table.style.setProperty('width', '100%'); // HEADER diff --git a/styles.css b/styles.css index 71cc60f..dd784bd 100644 --- a/styles.css +++ b/styles.css @@ -1,8 +1,3 @@ -/* - -This CSS file will be included with your plugin, and -available in the app when your plugin is enabled. - -If your plugin does not need CSS, delete this file. - -*/ +.sqlseal-table-container { + overflow-y: scroll; +} \ No newline at end of file