Merge branch 'main' into feature/frontmatterLinks

docs/.vitepress/config.mts

@@ -1,4 +1,5 @@
 import { defineConfig } from 'vitepress'
+import ohmMarkdownPlugin from '../plugins/ohm-plugin/vitepress-ohm-plugin'
 
 // https://vitepress.dev/reference/site-config
 export default defineConfig({
@@ -63,7 +64,9 @@ export default defineConfig({
       {
         text: 'FAQ',
         items: [
-          { text: 'Comparison with Dataview', link: '/faq/comparison-with-dataview' }
+          { text: 'Comparison with Dataview', link: '/faq/comparison-with-dataview' },
+		  { text: 'Understanding Tags', link: '/faq/understanding-tags'}
+
         ]
       },
       {
@@ -90,5 +93,10 @@ export default defineConfig({
       message: '',
       copyright: 'By <a href="https://hypersphere.blog">hypersphere</a>.<br/>Sponsor Me: <a href="https://ko-fi.com/hypersphere">Ko-Fi</a>'
     }
+  },
+  markdown: {
+    config(md) {
+      md.use(ohmMarkdownPlugin())
+    },
   }
 })

docs/.vitepress/theme/components/Stats.vue

@@ -1,22 +1,22 @@
 <script setup>
 const stats = [
   {
-    number: '2K+',
+    number: '3K+',
     label: 'Downloads',
     icon: '📥'
   },
   {
-    number: '48+',
+    number: '54+',
     label: 'Releases',
     icon: '📦'
   },
   {
-    number: '64+',
+    number: '88+',
     label: 'GitHub Stars',
     icon: '⭐'
   },
   {
-    number: '50+',
+    number: '99+',
     label: 'Discord Members',
     icon: '🤝'
   }

docs/.vitepress/theme/index.ts

@@ -3,6 +3,7 @@ import { h } from 'vue'
 import type { Theme } from 'vitepress'
 import DefaultTheme from 'vitepress/theme'
 import './style.css'
+import './syntax-highlighting.css'
 import Stats from './components/Stats.vue'
 
 export default {

docs/.vitepress/theme/style.css

@@ -133,3 +133,8 @@
   --docsearch-primary-color: var(--vp-c-brand-1) !important;
 }
 
+.video-wrapper {
+  margin: 1em auto;
+  display: grid;
+  justify-content: center;
+}

docs/.vitepress/theme/syntax-highlighting.css

@@ -0,0 +1,132 @@
+.language-sqlseal {
+
+    
+.vp-blockQuery {
+    --color: #297f30;
+}
+
+.vp-blockView {
+    --color: #c834dd;
+}
+
+.vp-blockFlag {
+    --color: #d08306;
+    font-weight: bold;
+    color: var(--color);
+}
+
+.vp-blockTable {
+    --color: #4985af;
+}
+
+.vp-identifier {
+    color: #2d7b33;
+}
+
+.vp-function {
+    color: #2c0e9b;
+}
+
+.vp-parameter {
+    color: #2c0e9b;
+}
+
+.vp-comment {
+    color: #7a7a7a;
+    font-style: italic;
+}
+
+.vp-keyword {
+    color: var(--color);
+    font-weight: bold;
+}
+
+.vp-literal {
+    color: rgb(0, 51, 255);
+}
+
+.vp-error {
+    color: rgb(235, 47, 47);
+    font-style: italic;
+}
+
+.vp-template-keyword {
+    color: #e3ab53;
+}
+
+    .vp-blockQuery, .vp-blockTable, .vp-blockFlag, .vp-blockView {
+        position: relative;
+    }
+
+    :is(.vp-blockQuery, .vp-blockTable, .vp-blockFlag, .vp-blockView)::before {
+        background-color: var(--color);
+        width: 5px;
+        top: -3px;
+        bottom: -5px;
+        /* left: 0; */
+        margin-left: -24px;
+        content: '';
+        display: block;
+        position: absolute;
+    }
+}
+
+.language-sqlseal {
+    display: block;
+    position: relative;
+    background: var(--vp-code-block-bg);
+    padding: 24px 0;
+    text-wrap: wrap;
+    clear: both;
+    overflow-y: scroll;
+    overflow-x: auto;
+}
+
+.dark .language-sqlseal {
+    color: #EEE;
+    --vp-code-block-color: #EEE;
+}
+
+.dark .vp-template-keyword {
+    color: #dd8e10;
+}
+
+.dark .vp-block-error {
+    color: red;
+}
+
+.dark .vp-blockQuery {
+    --color: #5eff6a;
+}
+
+.dark .vp-comment {
+    color: #898787;
+}
+
+.dark .vp-parameter {
+    color: #95ed22;
+}
+
+.dark .vp-literal {
+    color: yellow;
+}
+
+.dark .vp-blockView {
+    --color: #e945ff;
+}
+
+.dark .vp-blockFlag {
+    --color: #e3ab53;
+}
+
+.dark .vp-blockTable {
+    --color: #56b5fa;
+}
+
+.dark .vp-function {
+    color: #c4b8ef;
+}
+
+.dark .vp-identifier {
+    color: #6ff77a;
+}

docs/data-sources/vault-data.md

@@ -23,6 +23,8 @@ Files table consists of the following columns:
 | `file_size`         | Size of the file on disk (in bytes)                                                                                                                                              | 0.18.1        |
 | All file properties | All file properties are also added to the table. All the special characters will be transformed to underscores `_`, so for example `note type` will be accessible as `note_type` |               |
 
+> **Note:** Tags data can exist in two contexts - as file metadata here or as an entry in the tags table. Details [here](../faq/understanding-tags.md)
+
 ### `tags` table
 Tags table consists of the following columns:
 | Column   | Description                                                                                                              | Introduced In |
@@ -31,6 +33,8 @@ Tags table consists of the following columns:
 | `path`   | Full path of the file the tag belongs to                                                                                 | 0.24.1        |
 | `fileId` | (deprecated) same like `path`. Name changed for compatibility with other tables. Will get removed in the future versions |               |
 
+> **Note:** Tags data can in two contexts - as file metadata or as an entry in this tags table. Details [here](../faq/understanding-tags.md)
+
 ### `tasks` table
 Tasks table consists of the following columns:
 | Column      | Description                                                                                                              | Introduced In |
@@ -53,6 +57,7 @@ Introduced in 0.20.0
 | `display_text`  | Text displayed on the page for that link                           |
 | `target_exists` | information if the target file exists. 1 if it exists, 0 otherwise |
 
+
 #### Frontmatter links
 
 Links that appear in a file's frontmatter (Obsidian properties) contain a `frontmatterKey` property in the `position`
@@ -75,3 +80,4 @@ SELECT a(path) FROM links
 WHERE target = @path
 AND json_extract(position, '$.frontmatterKey') = 'type'
 ```
+

docs/faq/understanding-tags.md

@@ -0,0 +1,30 @@
+# Understanding Tags in SQLSeal
+When working with the SQLSeal plugin in Obsidian, tags can appear in two different contexts:
+
+- as a frontmatter property in your files
+- as rows in the `tags` table
+
+It is important to understand what to expect when querying them in these different contexts.
+
+## Tags in Frontmatter
+
+Obsidian’s frontmatter can include a `tags` field to declare tags directly within the YAML metadata at the top of a file.
+
+For example:
+```yaml
+---
+tags: [project, todo]
+---
+```
+
+These tags will appear as a list in the frontmatter and are considered a direct part of the file’s metadata.
+
+In the context of SQLSeal, they will be accessible when you query the files table, but they behave like any other frontmatter property-essentially as a plain list of values.
+
+## Tags in the tags Table
+
+The tags table is specifically designed to capture all tags found within the vault. Each tag in this table is represented as a separate row, along with the file path it’s associated with. For instance, if you have a tag #todo in multiple files, each instance of #todo will be recorded in the tags table, linking it to the respective file.
+
+## In Summary
+
+When using SQLSeal, keep in mind that tags from frontmatter and tags in the tags table serve different purposes and are structured differently. Frontmatter tags are essentially metadata fields of the file and appear as-is in the files table, while the tags table provides a detailed, row-based view of all tags in your vault.

docs/index.md

@@ -29,7 +29,7 @@ hero:
     <div class="content">
 Turn your Obsidian vault into a powerful, queryable database using SQL. SQLSeal seamlessly integrates with your notes and CSV and JSON files, enabling you to extract insights and patterns from your personal knowledge base.
 
-```sql
+```sqlseal
 SELECT * FROM files
 ORDER BY modified_at DESC
 LIMIT 10
@@ -71,7 +71,7 @@ LIMIT 10
 ## Query any data
 You can query any CSV and JSON data stored in your vault. SQLSeal can help you analyse your finances, workouts, goals and whatever you imagine!
 
-```sql
+```sqlseal
 TABLE commute = file(transport.csv)
 
 SELECT
@@ -97,9 +97,6 @@ GROUP BY Date, Transport_Type
 </div>
 
 
-
-
-
 <div class="vp-doc">
   <div class="container-split reverse">
 
@@ -122,6 +119,16 @@ SQLSeal Charts is currently being reviewed by Obsidian. As soon as it gets appro
 </div>
 
 
+## Learn more
+<p>This introduction video should help you get started with basics of SQLSeal</p>
+<div class="video-wrapper">
+  <iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/yXDrpWDeQPg?si=rttPk5LdH2lahqRN" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+</div>
+
+
+
+
+
 <Stats />
 
 

docs/links-and-images.md

@@ -5,14 +5,14 @@ SQLSeal allows for rendering links and images. For now the images needs to be ex
 
 ## Links
 To display a link, wrap use the `a` SQL function, for example:
-```sql
+```sqlseal
 SELECT a(path) FROM files LIMIT 10
 ```
 ![Example of links](./links.png)
 
 You can use second parameter to provide name for the link:
 
-```sql
+```sqlseal
 SELECT a(path, name) from files LIMIT 10
 ```
 
@@ -22,14 +22,14 @@ This API works for both filesystem and CSV files.
 ## Images
 You can embed images within your results. You need to wrap your resulting column with `img` function.
 
-```sql
+```sqlseal
 SELECT name, img(coverImg) FROM files
 ```
 
 ### Local images
 When using local images (stored in Obsidian) you need to pass second parameter being path of the original note. For example:
 
-```sql
+```sqlseal
 SELECT path, img(cover, path) FROM files
 ```
 
@@ -37,7 +37,7 @@ SELECT path, img(cover, path) FROM files
 The example below uses [Goodreads-books](https://www.kaggle.com/datasets/jealousleopard/goodreadsbooks) Kaggle dataset in CSV loaded in obsidian to display books with links to Open Library and showing the covers from Open Library Cover API. It uses Obsidian property to filter the author name.
 
 
-```sql
+```sqlseal
 TABLE books = file(books.csv)
 
 SELECT
@@ -56,6 +56,6 @@ LIMIT 10
 
 ## Checkboxes
 You can display boolean data as checkbox in the interface by calling `checkbox` function:
-```sql
+```sqlseal
 SELECT date, checkbox(excercised) FROM files WHERE date is not null
 ```

docs/plugins/ohm-plugin/vitepress-ohm-plugin.js

@@ -0,0 +1,275 @@
+// .vitepress/ohmMarkdownPlugin.js
+import * as ohm from 'ohm-js'
+import { SQLSealLangDefinition } from '../../../src/grammar/parser'
+import { highlighterOperation } from '../../../src/grammar/highlighterOperation'
+
+const parser = (sql) => {
+    const defaultViews = [
+        {
+            name: 'GRID',
+            singleLine: false,
+            argument: 'anyObject?'
+        },
+        {
+            name: 'HTML',
+            singleLine: true,
+            argument: 'restLine?'
+        },
+        {
+            name: 'MARKDOWN',
+            singleLine: true,
+            argument: 'restLine?'
+        },
+        {
+            name: 'LIST',
+            singleLine: true,
+            argument: 'restLine?'
+        },
+        {
+            name: 'TEMPLATE',
+            singleLine: false,
+            argument: 'handlebarsTemplate'
+        }
+    ]
+
+    const grammar = ohm.grammar(SQLSealLangDefinition(defaultViews, [], false))
+    const match = grammar.match(sql)
+    if (match.failed()) {
+        return []
+    }
+    const highlight = highlighterOperation(grammar)(match)
+
+    const results = highlight.highlight()
+    return results
+}
+
+/**
+ * Creates a markdown-it plugin that integrates Ohm.js parsing for code blocks
+ * 
+ * @returns {Function} A markdown-it plugin function
+ */
+export default function ohmMarkdownPlugin() {
+    // Languages we want to parse with Ohm.js
+    const supportedLanguages = ['js', 'javascript', 'ts', 'typescript', 'jsx', 'tsx', 'sqlseal']
+    
+    // Track processed tokens to prevent duplicate processing
+    const processedTokens = new WeakSet()
+
+    /**
+     * This is the actual markdown-it plugin function that gets registered
+     * with markdown-it.use()
+     */
+    return (md) => {
+        // Store the original fence renderer
+        const originalFence = md.renderer.rules.fence
+
+        // Replace it with our custom implementation
+        md.renderer.rules.fence = (tokens, idx, options, env, slf) => {
+            const token = tokens[idx]
+            
+            // Check if we've already processed this token to prevent duplicate processing
+            if (processedTokens.has(token)) {
+                return originalFence(tokens, idx, options, env, slf)
+            }
+            
+            // Mark this token as processed
+            processedTokens.add(token)
+            
+            const code = token.content
+            const lang = token.info.trim()
+
+            // Skip unsupported languages
+            if (!supportedLanguages.includes(lang)) {
+                return originalFence(tokens, idx, options, env, slf)
+            }
+
+            try {
+                // Parse the code with Ohm.js
+                const rawDecorators = parser(code)
+
+                // If no decorators found, use the original renderer
+                if (!rawDecorators || rawDecorators.length === 0) {
+                    return originalFence(tokens, idx, options, env, slf)
+                }
+
+                // Process decorators with block types prioritized to create nested structure
+                const html = buildNestedDecorators(code, rawDecorators, lang)
+                
+                return html
+
+            } catch (error) {
+                console.error('❌ Error parsing code with Ohm.js:', error)
+                return originalFence(tokens, idx, options, env, slf)
+            }
+        }
+    }
+}
+
+/**
+ * Builds nested HTML structure where block decorators properly
+ * encapsulate smaller decorators
+ * 
+ * @param {string} code - The code to be decorated
+ * @param {Array} decorators - The decorators to apply
+ * @param {string} lang - The language of the code
+ * @returns {string} HTML with properly nested decorators
+ */
+function buildNestedDecorators(code, decorators, lang) {
+    if (!decorators || decorators.length === 0) {
+        return `<pre class="language-${lang} vp-custom-code-block"><code class="language-${lang}">${escapeHtml(code)}</code></pre>`
+    }
+    
+    // Sort decorators by priority (block types first) and then by size (largest first)
+    const sortedDecorators = [...decorators].sort((a, b) => {
+        // First prioritize block types
+        const aIsBlock = a.type.startsWith('block')
+        const bIsBlock = b.type.startsWith('block')
+        
+        if (aIsBlock && !bIsBlock) return -1
+        if (!aIsBlock && bIsBlock) return 1
+        
+        // If both are blocks or both are not blocks, sort by size (larger first)
+        const aSize = a.end - a.start
+        const bSize = b.end - b.start
+        
+        return bSize - aSize
+    })
+    
+    // Create a tree representation of the decorators
+    const decoratorTree = buildDecoratorTree(sortedDecorators)
+    
+    // Render the HTML with the tree
+    let html = `<pre class="language-${lang} vp-custom-code-block"><code class="language-${lang}">`
+    html += renderDecoratorTree(decoratorTree, code)
+    html += '</code></pre>'
+    
+    return html
+}
+
+/**
+ * Builds a tree representation of decorators where parent nodes
+ * contain child nodes
+ * 
+ * @param {Array} decorators - Sorted decorators array
+ * @returns {Array} Tree structure of decorators
+ */
+function buildDecoratorTree(decorators) {
+    // Initialize with the entire code range as the root
+    const root = {
+        start: 0,
+        end: Infinity,
+        children: []
+    }
+    
+    // Function to add a decorator to the tree
+    function addToTree(node, decorator) {
+        // Check if this decorator fits within any of the children
+        for (let i = 0; i < node.children.length; i++) {
+            const child = node.children[i]
+            
+            // If the decorator fits entirely within this child
+            if (decorator.start >= child.start && decorator.end <= child.end) {
+                return addToTree(child, decorator)
+            }
+        }
+        
+        // This decorator doesn't fit within any child, so it's a direct child of this node
+        const newNode = {
+            ...decorator,
+            children: []
+        }
+        
+        // Any existing children that fit within this new node should become its children
+        for (let i = node.children.length - 1; i >= 0; i--) {
+            const child = node.children[i]
+            
+            if (child.start >= decorator.start && child.end <= decorator.end) {
+                newNode.children.push(child)
+                node.children.splice(i, 1)
+            }
+        }
+        
+        // Sort the children by start position
+        newNode.children.sort((a, b) => a.start - b.start)
+        
+        // Add the new node to the current node's children
+        node.children.push(newNode)
+        
+        // Sort the children by start position
+        node.children.sort((a, b) => a.start - b.start)
+    }
+    
+    // Add each decorator to the tree
+    for (const decorator of decorators) {
+        addToTree(root, decorator)
+    }
+    
+    return root
+}
+
+/**
+ * Renders the decorator tree as HTML
+ * 
+ * @param {Object} node - Current tree node
+ * @param {string} code - The original code
+ * @param {number} lastPos - Last rendered position in the code
+ * @returns {string} HTML representation of the tree
+ */
+function renderDecoratorTree(node, code, lastPos = 0) {
+    let html = ''
+    let currentPos = lastPos
+    
+    // If this is not the root node, add its content
+    if (node.type) {
+        html += `<span class="vp-code-decorator vp-${node.type}" data-type="${node.type}">`
+    }
+    
+    // Add the children with the text between them
+    for (const child of node.children) {
+        // Add text before this child
+        if (child.start > currentPos) {
+            html += escapeHtml(code.substring(currentPos, child.start))
+        }
+        
+        // Add the child
+        html += renderDecoratorTree(child, code, child.start)
+        
+        // Update the current position
+        currentPos = child.end
+    }
+    
+    // Add the remaining text in this node
+    const endPos = node.type ? node.end : code.length
+    if (currentPos < endPos) {
+        html += escapeHtml(code.substring(currentPos, endPos))
+    }
+    
+    // Close the tag if this is not the root
+    if (node.type) {
+        html += '</span>'
+    }
+    
+    return html
+}
+
+/**
+ * Escapes HTML and additionally escapes handlebars/mustache syntax 
+ * to prevent Vue from interpreting them as Vue template directives
+ */
+function escapeHtml(unsafe) {
+    // First do regular HTML escaping
+    let escaped = unsafe
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;")
+        .replace(/'/g, "&#039;");
+    
+    // Then escape the handlebars/mustache syntax by adding a zero-width space
+    // between the opening braces to prevent Vue from interpreting them
+    escaped = escaped
+        .replace(/{{/g, "{&#8203;{")
+        .replace(/}}/g, "}&#8203;}");
+    
+    return escaped;
+}

docs/renderers/template.md

@@ -16,7 +16,7 @@ The following variables are exposed
 
 
 ## Example
-```sql
+```sqlseal
 TEMPLATE
 Current Path: {{properties.path}}
 {{#each data}}

docs/using-properties.md

@@ -15,11 +15,11 @@ On top of properties current file exposes in the properties, you can also use th
 
 ### Example 1: Selecting just the current file from the files table
 
-```sql
+```sqlseal
 SELECT * FROM files WHERE path = @path
 ```
 
 ### Example2: Select tags of the current file
-```sql
+```sqlseal
 SELECT * FROM tags WHERE path = @path
 ```

src/grammar/highlighterOperation.ts

@@ -54,7 +54,10 @@ export const highlighterOperation = (grammar: ohm.Grammar) => {
 
     s.addOperation<any>('highlight', {
         _terminal() {
-            return computeNodeHighlight(this)
+            return [
+                ...computeNodeHighlight(this),
+                ...this.children.map(c => c.highlight()).flat()
+            ]
         },
         _nonterminal() {
             return [
@@ -74,6 +77,13 @@ export const highlighterOperation = (grammar: ohm.Grammar) => {
             }
             return [...computeNodeHighlight(this), ...results]
         },
+        comment(_comment) {
+            return [{
+                type: 'comment',
+                start: this.source.startIdx,
+                end: this.source.endIdx
+            }]
+        },
         errorLine: (node, _nl) => {
             return [{
                 type: 'error',

src/renderer/ListRenderer.ts

@@ -51,11 +51,12 @@ export class ListRenderer implements RendererConfig {
                 })
 
                 data.forEach((d: any) => {
-                    const row = list.createEl("li", { cls: ['sqlseal-list-element'] }).createEl('ul')
+					const singleCol = columns.length == 1; // Only one column, do not nest lists
+                    const row = singleCol ? list : list.createEl("li", { cls: ['sqlseal-list-element'] }).createEl('ul')
                     columns.forEach((c: any) => {
                         const el = row.createEl("li", {
                             text: createEl('span', { text: c, cls: 'sqlseal-column-name' }) as any, // FIXME: this should be properly typed
-                            cls: ['sqlseal-list-element-single']
+                            cls: singleCol ? ['sqlseal-list-element', 'sqlseal-list-element-single'] : ['sqlseal-list-element-single']
                         })
                         const val = this.cellParser.render(d[c])
                         el.append(val)
@@ -68,4 +69,4 @@ export class ListRenderer implements RendererConfig {
             }
         }
     }
-}
+}

src/vaultSync/tables/filesTable.ts

@@ -17,7 +17,7 @@ const extractFrontmatterFromFile = (file: TFile, plugin: Plugin): Record<string,
     )
 }
 
-function fileData(file: TFile, { tags: _tags, ...frontmatter }: Record<string, any>) {
+function fileData(file: TFile, { ...frontmatter }: Record<string, any>) {
     return {
         ...frontmatter,
         id: file.path,
@@ -89,4 +89,4 @@ export class FilesFileSyncTable extends AFileSyncTable {
             this.db.createIndex(`files_${column}_idx`, FILES_TABLE_NAME, [column])
         ))
     }
-}
+}

tsconfig.json

@@ -25,6 +25,5 @@
     "allowSyntheticDefaultImports": true
   },
   "include": [
-    "**/*.ts"
-  ]
+    "**/*.ts"]
 }