feat: improving file tables sync. added better way of extending library with new data types.

src/codeblockHandler/CodeblockProcessor.ts

@@ -29,7 +29,7 @@ export class CodeblockProcessor extends MarkdownRenderChild {
 
     async onload() {
         try {
-            const results = parseLanguage(this.source)
+            const results = parseLanguage(this.source, this.ctx.sourcePath)
             if (results.tables) {
                 await this.registerTables(results.tables)
                 if (!results.queryPart) {
@@ -89,9 +89,7 @@ export class CodeblockProcessor extends MarkdownRenderChild {
                 
             } else {
                 path = this.app.vault.getFileByPath(this.ctx.sourcePath)
-                extras = {
-                    tableNo: table.fileName // FIXME: this should be returned under better name.
-                }
+                extras = table?.extras ?? {}
             }
 
             if (!path) {

src/database/database.ts

@@ -84,6 +84,10 @@ export class SqlSealDatabase {
         this.isConnected = false
     }
 
+    async recreateDatabase() {
+        await this.db.recreateDatabase()
+    }
+
     async createTableWithData(name: string, data: Array<Record<string, unknown>>) {
         const schema = await this.getSchema(data)
         await this.createTable(name, schema)
@@ -119,6 +123,11 @@ export class SqlSealDatabase {
     async createTable(name: string, fields: Record<string, FieldTypes>, noDrop?: boolean) {
         await this.db.createTable(name, fields, noDrop)
     }
+
+    async createTableNoTypes(name: string, columns: string[], noDrop?: boolean) {
+        await this.db.createTableNoTypes(name, columns, noDrop)
+    }
+
     async getSchema(data: Array<Record<string, unknown>>) {
         const fields = Object.keys(data.reduce((acc, obj) => ({ ...acc, ...obj }), {}));
         const { types } = toTypeStatements(fields, data as Array<Record<string, string>>); // Call the toTypeStatements function with the correct arguments

src/database/worker/database.ts

@@ -6,6 +6,7 @@ import IndexedDBBackend from '../../../node_modules/absurd-sql/dist/indexeddb-ba
 import type { BindParams, Database, Statement } from "sql.js";
 import { sanitise } from "../../utils/sanitiseColumn";
 import { FieldTypes } from "../../utils/typePredictions";
+import { uniq } from "lodash";
 
 
 function toObjectArray(stmt: Statement) {
@@ -112,6 +113,16 @@ export class WorkerDatabase {
         })
     }
 
+    async recreateDatabase() {
+        this.db.exec(`
+            PRAGMA writable_schema = 1;
+            DELETE FROM sqlite_master;
+            PRAGMA writable_schema = 0;
+            VACUUM;
+            PRAGMA integrity_check;
+`)
+    }
+
     async createTable(tableName: string, fields:  Record<string, FieldTypes>, noDrop: boolean = false) {
         const transformedFiels = Object.entries(fields).map(([key, type]) => [sanitise(key), type])
         const uniqueFields = [...new Map(transformedFiels.map(item =>
@@ -132,6 +143,16 @@ export class WorkerDatabase {
         // Dropping data.
     }
 
+    /* Types are optional in SQLite, we can take advantage of that */
+    async createTableNoTypes(tableName: string, columns: string[], noDrop: boolean = false) {
+        const fields = uniq(columns.map(f => sanitise(f)))
+        if (!noDrop) {
+            await this.dropTable(tableName)
+        }
+        const createStmt = `CREATE TABLE IF NOT EXISTS ${tableName}(${fields.join(',')})`
+        this.db.prepare(createStmt).run()
+    }
+
     async clearTable(tableName: string) {
         this.db.prepare(`DELETE FROM ${tableName}`).run()
 

src/datamodel/repository/configuration.ts

@@ -0,0 +1,28 @@
+import { Repository } from "./abstractRepository";
+
+export class ConfigurationRepository extends Repository {
+    async init() {
+        await this.createTable()
+    }
+
+    private async createTable() {
+        await this.db.createTableNoTypes('configuration', ['id', 'value'], true)
+    }
+
+    public async getConfig(key: string) {
+        try {
+            const config = await this.db.select('SELECT value FROM configuration WHERE id = @id', { id: key })
+            if (config.data && config.data.length) {
+                return config.data[0].value
+            }
+            throw new Error(`Configuration value not found for key: ${key}`)
+        } catch (e) {
+            throw e
+        }
+    }
+
+    public async setConfig(key: string, value: number | string) {
+       await this.db.deleteData('configuration', [{ id: key }], 'id')
+       await this.db.insertData('configuration', [{ id: key, value }])
+    }
+}

src/datamodel/sync.ts

@@ -91,7 +91,6 @@ export class Sync {
             const syncObject = SyncStrategyFactory.getStrategyByFileLog(entry, this.app)
 
             const { data, columns } = await syncObject.returnData()
-            console.log('GOT NEW DATA FOR', entry, data, columns)
 
             const tableName = entry.table_name
             await this.db.createTableClean(tableName, columns)

src/datamodel/syncStrategy/CsvFileSyncStrategy.ts

@@ -1,12 +1,14 @@
-import { App } from "obsidian";
+import { App, Component } from "obsidian";
 import { TableRegistration } from "../types";
 import { ISyncStrategy } from "./abstractSyncStrategy";
 import { sanitise } from "src/utils/sanitiseColumn";
 import { parse } from "papaparse";
 import { FieldTypes, toTypeStatements } from "src/utils/typePredictions";
 import { FilepathHasher } from "src/utils/hasher";
+import { TableDefinitionConfig } from "./types";
+import { SourceType } from "src/grammar/newParser";
 
-export class FileSyncStrategy implements ISyncStrategy {
+export class CsvFileSyncStrategy implements ISyncStrategy {
     constructor(private reg: TableRegistration, private app: App) {
 
     }
@@ -16,6 +18,14 @@ export class FileSyncStrategy implements ISyncStrategy {
         return `file_${hash}`
     }
 
+    static processTableDefinition(config: TableDefinitionConfig) {
+            return {
+                tableName: config.alias,
+                type: config.type as SourceType,
+                fileName: config.arguments
+            }
+        }
+
     async returnData() {
         const file = this.app.vault.getFileByPath(this.reg.sourceFile)!
         const data = await this.app.vault.cachedRead(file)

src/datamodel/syncStrategy/MarkdownTableSyncStrategy.ts

@@ -1,8 +1,10 @@
 import { FilepathHasher } from "src/utils/hasher";
 import { TableRegistration } from "../types";
 import { ISyncStrategy } from "./abstractSyncStrategy";
-import { App, MarkdownRenderer, TFile } from "obsidian";
+import { App, Component, MarkdownRenderer, TFile } from "obsidian";
 import { FieldTypes, toTypeStatements } from "src/utils/typePredictions";
+import { TableDefinitionConfig } from "./types";
+import { SourceType } from "src/grammar/newParser";
 
 
 interface TableData {
@@ -10,11 +12,22 @@ interface TableData {
     data: Array<Record<string, string>>;
 }
 
-export class TableSyncStrategy implements ISyncStrategy {
+export class MarkdownTableSyncStrategy implements ISyncStrategy {
     constructor(private reg: TableRegistration, private app: App) {
 
     }
 
+    static processTableDefinition(config: TableDefinitionConfig) {
+        return {
+            tableName: config.alias,
+            type: config.type as SourceType,
+            fileName: config.sourceFile, //+ '?table=' + config.arguments
+            extras: {
+                tableNo: parseInt(config.arguments, 10) ?? 0
+            }
+        }
+    }
+
     async tableName() {
         const hash = await FilepathHasher.sha256(`${this.reg.sourceFile}__${this.reg.fileName}`) // FILENAME is in this case 
 
@@ -24,16 +37,16 @@ export class TableSyncStrategy implements ISyncStrategy {
     async returnData() {
 
         const tableIndex = this.reg.extras.tableNo
-        console.log('TABLE INDEX', tableIndex)
         const file = this.app.vault.getFileByPath(this.reg.sourceFile)!
-        // Get the file content
         const content = await this.app.vault.read(file);
 
         // Create a temporary div to render the markdown
         const tempDiv = document.createElement('div');
 
         // Use Obsidian's MarkdownRenderer to parse the content
-        await MarkdownRenderer.renderMarkdown(content, tempDiv, file.path, null);
+        const component = new Component()
+
+        await MarkdownRenderer.render(this.app, content, tempDiv, file.path, component)
 
         // Find all tables in the rendered content
         const tables = tempDiv.querySelectorAll('table');

src/datamodel/syncStrategy/SyncStrategyFactory.ts

@@ -2,8 +2,8 @@ import { App } from "obsidian";
 import { FileLog } from "../repository/fileLog";
 import { TableRegistration } from "../types";
 import { ISyncStrategy } from "./abstractSyncStrategy";
-import { FileSyncStrategy } from "./FileSyncStrategy";
-import { TableSyncStrategy } from "./TableSyncStrategy";
+import { CsvFileSyncStrategy } from "./CsvFileSyncStrategy";
+import { MarkdownTableSyncStrategy } from "./MarkdownTableSyncStrategy";
 
 const fileLogToTableRegistration = (log: FileLog): TableRegistration => {
     return {
@@ -15,13 +15,35 @@ const fileLogToTableRegistration = (log: FileLog): TableRegistration => {
     }
 }
 
+const resolveFileStrategy = (filename: string) => {
+    const parts = filename.split('.')
+    const extension = parts[parts.length - 1].toLowerCase()
+
+    switch (extension) {
+        case 'csv':
+            return CsvFileSyncStrategy
+        default:
+            throw new Error(`No file processor for extension ${extension}`)
+    }
+}
+
 export class SyncStrategyFactory {
     static getStrategy(reg: TableRegistration, app: App): ISyncStrategy {
         switch (reg.type) {
             case 'file':
-                return new FileSyncStrategy(reg, app)
+                const Cls = resolveFileStrategy(reg.fileName)
+                return new Cls(reg, app)
             case 'table':
-                return new TableSyncStrategy(reg, app)
+                return new MarkdownTableSyncStrategy(reg, app)
+        }
+    }
+
+    static getStaticStrategyReference(type: string) {
+        switch (type) {
+            case 'csv-file':
+                return CsvFileSyncStrategy
+            case 'table':
+                return MarkdownTableSyncStrategy
         }
     }
 

src/datamodel/syncStrategy/types.ts

@@ -0,0 +1,6 @@
+export interface TableDefinitionConfig {
+    type: string;
+    alias: string;
+    arguments: string;
+    sourceFile: string;
+}

src/grammar/newParser.ts

@@ -7,6 +7,7 @@ export interface Table {
     tableName: string;
     type: SourceType,
     fileName: string;
+    extras?: Record<string, string | number>
 }
 
 export interface TableWithParentPath extends Table {