feat: ability to query vault data finished (POC)

CHANGELOG.md

@@ -0,0 +1,7 @@
+# 0.2.0
+- Added ability to query files in the fault directly.
+- Added observability - when CSV or file in the vault is changed, all SELECTS that uses it should update too
+- Added custom class to sqlseal tables and ability to scroll vertically when the data is overflowing horizontally. 
+
+# 0.1.0
+- Initial release. Allows to create tables based on CSV files in your vault and query them using SQL.

docs/.vitepress/config.mts

@@ -18,8 +18,8 @@ export default defineConfig({
         items: [
           { text: 'Quick Start', link: '/quick-start' },
           { text: 'Using properties', link: '/using-properties' },
-          { text: 'Future Plans', link: '/future-plans' }
-
+          { text: 'Future Plans', link: '/future-plans' },
+          { text: 'Query Vault Content', link: '/query-vault-content' },
         ]
       }
     ],
@@ -29,7 +29,7 @@ export default defineConfig({
     ],
     footer: {
       message: '',
-      copyright: 'By <a href="https://hypersphere.blog">hypersphere</a>'
+      copyright: 'By <a href="https://hypersphere.blog">hypersphere</a>. Ko-Fi: <a href="https://ko-fi.com/hypersphere">hypersphere</a>'
     }
   }
 })

docs/index.md

@@ -16,7 +16,7 @@ features:
     details: Use full power of SQL to select, join, filter data for your liking
   - title: Fully featured SQL Engine
     details: With SQLite under the hood, you can use all functionality of the database
-  - title: Zero configuration
-    details: Just use SQL syntax to create and query the data
+  - title: Query your files and tags
+    details: Use SQL to filter files in your vault
 ---
 

docs/query-vault-content.md

@@ -0,0 +1,27 @@
+# Query your Vault content
+
+You can query your vault content using SQLSeal built-in tables `files` and `tags`. You can use them to query specific files in the fault based on Properties (Frontmatter) and associated tags.
+
+## Example: Get all files from the vault
+To get all files from the fault you can run the following query:
+
+```sqlseal
+SELECT * FROM files
+```
+
+## Filter by Properties
+If your files have frontmatter properties, you can query by them using SQL `WHERE` clause. SQLSeal automatically maintains SQL schema and creates columns when needed. Let's assume we have files with property `type`. We can query only specific notes by running the following:
+
+```sqlseal
+SELECT * FROM files WHERE type = 'resource'
+```
+
+The query above will return only files that have property `type` set to value `resource`.
+
+## Filter by Tags
+Tags are kept in a separate table `tags`. To select all files that have specific tag, we can perform simple join.
+
+```sqlseal
+SELECT files.* FROM files JOIN tags ON files.id=tags.fileId WHERE tag = '#important'
+```
+

main.ts

@@ -1,4 +1,5 @@
 import { Plugin } from 'obsidian';
+import { SealFileSync } from 'src/SealFileSync';
 import { SqlSealSettings } from 'src/settings';
 import { SqlSeal } from 'src/sqlSeal';
 
@@ -6,28 +7,33 @@ const DEFAULT_SETTINGS = { rows: [] }
 
 export default class SqlSealPlugin extends Plugin {
 	settings: SqlSealSettings;
+	fileSync: SealFileSync;
+	sqlSeal: SqlSeal;
 
 	async onload() {
 		await this.loadSettings();
 		const sqlSeal = new SqlSeal(this.app, false)
 		this.registerMarkdownCodeBlockProcessor("sqlseal",  sqlSeal.getHandler())
 		await sqlSeal.connect()
+		
+		this.sqlSeal = sqlSeal
 
 
-		// const fileSync = new SealFileSync(this.app, sqlSeal)
-		// setTimeout(() => {
-		// 	fileSync.init()
-		// }, 5000)
+		this.fileSync = new SealFileSync(this.app, sqlSeal, this)
+		// start syncing when files are loaded
+		this.app.workspace.onLayoutReady(() => {
+			this.fileSync.init()
+		})
 
 	// this.addSettingTab(new SqlSealSettingsTab(this.app, this));
 
-
-
-
 	}
 
 	onunload() {
-
+		if (this.fileSync) {
+			this.fileSync.destroy();
+		}
+		this.sqlSeal.db.disconect();
 	}
 
 	async loadSettings() {

manifest.json

@@ -1,11 +1,11 @@
 {
 	"id": "hypersphere-sqlseal",
 	"name": "SQL Seal",
-	"version": "0.1.0",
+	"version": "0.2.0",
 	"minAppVersion": "0.15.0",
-	"description": "Use SQL in your notes!",
+	"description": "Use SQL in your notes to query your vault and CSV files.",
 	"author": "hypersphere",
 	"authorUrl": "https://hypersphere.blog/",
-	"fundingUrl": "https://hypersphere.blog",
+	"fundingUrl": "ko-fi.com/hypersphere",
 	"isDesktopOnly": false
 }

package.json

@@ -1,7 +1,7 @@
 
 {
   "name": "obsidian-sqlseal",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "A plugin for Obsidian that allows you to run SQL queries on your notes.",
   "main": "main.js",
   "scripts": {

src/SealFileSync.ts

@@ -1,12 +1,14 @@
-import { App, TAbstractFile } from "obsidian";
+import { App, EventRef, Plugin, TAbstractFile, TFile } from "obsidian";
 import GrayMatter from "gray-matter";
 import { SqlSeal } from "./sqlSeal";
+import { delay } from "./utils";
 
 
 
 async function extractFrontmatterFromFile(file: TAbstractFile) {
     const content = await this.app.vault.read(file);
     const gm = GrayMatter(content)
+
     return gm.data
 }
 
@@ -21,36 +23,144 @@ function fileData(file: TAbstractFile, frontmatter: Record<string, any>) {
 
 export class SealFileSync {
     private currentSchema: Record<string, 'TEXT' | 'INTEGER' | 'REAL'> = {}
-    constructor(public readonly app: App, private readonly sqlSeal: SqlSeal) {
-        this.app.vault.on('modify', async (file) => {
-            console.log('Modified', file)
-            const frontmatter = await extractFrontmatterFromFile(file)
-            if (this.isFrontmatterSame([frontmatter])) {
-                // we need to update the row
-                await this.sqlSeal.db.updateData('files', [fileData(file, frontmatter)])
+    private refs: Array<EventRef> = []
+    constructor(public readonly app: App, private readonly sqlSeal: SqlSeal, private readonly plugin: Plugin) {
+        this.refs.push(this.app.vault.on('modify', async (file) => {
+            if (!(file instanceof TFile)) {
                 return
             }
-            await this.init()
+            console.log('Modified', file)
+            const frontmatter = await extractFrontmatterFromFile(file)
+
+            if (this.hasNewColumns(frontmatter)) {
+                await delay(1000)
+                await this.init()
+
+                return
+            }
+            // we need to update the row
+            await this.sqlSeal.db.updateData('files', [fileData(file, frontmatter)])
+            await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId')
+
+            this.sqlSeal.observer.fireObservers('table:files')
+
+
+            // Wait 1 second before updating tags table
+            await delay(1000)
+            await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
+            this.sqlSeal.observer.fireObservers('table:tags')
+        }))
+
+        this.refs.push(this.app.vault.on('create', async (file) => {
+            if (!(file instanceof TFile)) {
+                return
+            }
+            console.log('File created', file.path)
+            const frontmatter = await extractFrontmatterFromFile(file)
+
+
+            if (this.hasNewColumns(frontmatter)) {
+                await delay(1000)
+                await this.init()
+
+                return
+            }
+
+
+            // we need to update the row
+            await this.sqlSeal.db.insertData('files', [fileData(file, frontmatter)])
+            this.sqlSeal.observer.fireObservers('table:files')
+
+            // Wait 1 second before updating tags table
+            await delay(1000)
+            await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
+            this.sqlSeal.observer.fireObservers('table:tags')
+        }))
+        
+        this.refs.push(this.app.vault.on('delete', async (file) => {
+            if (!(file instanceof TFile)) {
+                return
+            }
+
+            console.log('File deleted', file.path)
+            await this.sqlSeal.db.deleteData('files', [{ id: file.path }])
+            this.sqlSeal.observer.fireObservers('table:files')
+
+            await this.sqlSeal.db.deleteData('tags', [{ fileId: file.path }], 'fileId')
+            this.sqlSeal.observer.fireObservers('table:tags')
+        }))
+
+        this.refs.push(this.app.vault.on('rename', async (file, oldPath) => {
+            if (!(file instanceof TFile)) {
+                return
+            }
+
+            console.log('File renamed', file.path)
+            // deleting old one and adding new one
+            await this.sqlSeal.db.deleteData('files', [{ id: oldPath }])
+
+            // delete old tags
+            await this.sqlSeal.db.deleteData('tags', [{ fileId: oldPath }], 'fileId')
+
+            await this.sqlSeal.db.insertData('files', [fileData(file, await extractFrontmatterFromFile(file))])
+            this.sqlSeal.observer.fireObservers('table:files')
+
+            // Wait 1 second before updating tags table
+            await delay(1000)
+            await this.sqlSeal.db.insertData('tags', await this.getFileTags(file))
+            this.sqlSeal.observer.fireObservers('table:tags')
+
+        }))
+
+        // add Obsidian command to reload SQLSeal file database
+        this.plugin.addCommand({
+            id: 'reload-sqlseal',
+            name: 'Reload Vault Database',
+            callback: async () => {
+                await this.init()
+            }
         })
     }
+
+    destroy() {
+        this.refs.forEach(ref => this.app.vault.offref(ref))
+    }
+
+    async getFileTags(file: TFile) {
+
+        // Get fresh tag
+        console.log('FILE', file)
+
+
+        return (this.app.metadataCache.getFileCache(file)?.tags || [])
+            .map(t => t.tag)
+            .map(t => ({ tag: t, fileId: file.path }))
+    }
+
     async init() {
         const files = this.app.vault.getMarkdownFiles();
 
         const data = []
+        const tags: Array<{fileId: string, tag: string }> = []
 
         for (const file of files) {
             const frontmatter = await extractFrontmatterFromFile(file)
+            tags.push(...await this.getFileTags(file))
             data.push(fileData(file, frontmatter))
         }
 
-
         const schema = await this.sqlSeal.db.createTableWithData('files', data)
         this.currentSchema = schema
+        await this.sqlSeal.db.createTableWithData('tags', tags)
+        this.sqlSeal.observer.fireObservers('table:files')
+        this.sqlSeal.observer.fireObservers('table:tags')
     }
 
-    isFrontmatterSame(newFrontmatter: Array<Record<string, any>>) {
-        const newSchema = this.sqlSeal.db.getSchema(newFrontmatter)
-        return JSON.stringify(newSchema) === JSON.stringify(this.currentSchema)        
+    hasNewColumns(newFrontmatter: Record<string, any>) {
+        const currentFields = Object.keys(this.currentSchema)
+        const newFields = Object.keys(newFrontmatter).filter(f => !currentFields.includes(f))
 
+        console.log('current fields', currentFields, Object.keys(newFrontmatter))
+        return newFields.length > 0
     }
 }

src/database.ts

@@ -105,6 +105,14 @@ export class SqlSealDatabase {
         this.connectingPromiseResolve()
     }
 
+    async disconect() {
+        if (!this.isConnected) {
+            return
+        }
+        this.db.close()
+        this.isConnected = false
+    }
+
     async createTableWithData(name: string, data: Array<Record<string, unknown>>) {
         const schema = await this.getSchema(data)
         await this.createTable(name, schema)
@@ -113,6 +121,20 @@ export class SqlSealDatabase {
         return schema
     }
 
+    async addNewColumns(name: string, data: Array<Record<string, unknown>>) {
+        const schema = await this.getSchema(data)
+        const currentSchema = this.db.prepare(`PRAGMA table_info(${name})`).all()
+        const currentFields = currentSchema.map((f: any) => f.name)
+        const newFields = Object.keys(schema).filter(f => !currentFields.includes(f))
+
+        if (newFields.length === 0) {
+            return
+        }
+
+        const alter = this.db.prepare(`ALTER TABLE ${name} ADD COLUMN ${newFields.map(f => `${f} ${schema[f]}`).join(', ')}`)
+        alter.run()
+    }
+
     updateData(name: string, data: Array<Record<string, unknown>>) {
         const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
         const update = this.db.prepare(`UPDATE ${name} SET ${fields.map((key: string) => `${key} = @${key}`).join(', ')} WHERE id = @id`);
@@ -129,6 +151,22 @@ export class SqlSealDatabase {
         return updateMany(data)
     }
 
+    deleteData(name: string, data: Array<Record<string, unknown>>, key: string = 'id') {
+
+        const deleteStmt = this.db.prepare(`DELETE FROM ${name} WHERE ${key} = @${key}`);
+        const deleteMany = this.db.transaction((pData: Array<Record<string, any>>) => {
+            pData.forEach(data => {
+                try {
+                    deleteStmt.run(data)
+                } catch (e) {
+                    console.log(e)
+                }
+            })
+        })
+
+        return deleteMany(data)
+    }
+
     insertData(name: string, data: Array<Record<string, unknown>>) {
         const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
         const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`);
@@ -201,7 +239,7 @@ export class SqlSealDatabase {
     }
 
     async defineDatabaseFromUrl(unprefixedName: string, url: string, prefix: string, reloadData: boolean = false) {
-        const name = prefixedIfNotGlobal(unprefixedName, [], prefix)
+        const name = prefixedIfNotGlobal(unprefixedName, [], prefix) // FIXME: should we pass global tables here too?
         if (this.savedDatabases[name]) {
             console.log('Database Exists', name)
             if (reloadData) {

src/sqlSeal.ts

@@ -9,17 +9,25 @@ import { delay } from "./utils";
 
 export class SqlSeal {
     public db: SqlSealDatabase
-    private observer: SealObserver
+    public observer: SealObserver
     constructor(private readonly app: App, verbose = false) {
         this.db = new SqlSealDatabase(app, verbose)
         this.observer = new SealObserver(verbose)
         this.observeAllFileChanges()
     }
 
+    public get globalTables() {
+        return ['files', 'tags']
+    }
+
     async connect() {
         await this.db.connect()
     }
 
+    async disconnect() {
+        await this.db.disconect()
+    }
+
     async resolveFrontmatter(ctx: MarkdownPostProcessorContext) {
         if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) {
             return ctx.frontmatter as Record<string, any>
@@ -73,7 +81,7 @@ export class SqlSeal {
     if (selectMatch) {
         try {
             const selectStatement = selectMatch[0]
-            const { statement, tables } = updateTables(selectStatement, ['files'], prefix)
+            const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix)
 
 
             const renderSelect = async () => {

src/ui.ts

@@ -1,6 +1,9 @@
 export const displayData = (el: HTMLElement, columns, data) => {
     el.empty()
-    const table = el.createEl("table")
+    const container = el.createDiv({
+        cls: 'sqlseal-table-container'
+    })
+    const table = container.createEl("table")
     table.style.setProperty('width', '100%');
 
     // HEADER

styles.css

@@ -1,8 +1,3 @@
-/*
-
-This CSS file will be included with your plugin, and
-available in the app when your plugin is enabled.
-
-If your plugin does not need CSS, delete this file.
-
-*/
+.sqlseal-table-container {
+    overflow-y: scroll;
+}