diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/docs.yml similarity index 100% rename from .github/workflows/deploy-docs.yml rename to .github/workflows/docs.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..40efac8 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,40 @@ +name: Release Obsidian plugin + +on: + push: + tags: + - "*" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: "18.x" + + - name: Build plugin + run: | + npm install + npm run build + - name: Recompile Better SQLite3 for Electron + run: | + npm run rebuild + - name: Copy better SQLite3 to plugin + run: | + cp -r node_modules/better-sqlite3/build/Release/better_sqlite3.node . + + - name: Create release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tag="${GITHUB_REF#refs/tags/}" + + gh release create "$tag" \ + --title="$tag" \ + --draft \ + main.js manifest.json styles.css better_sqlite3.node \ No newline at end of file diff --git a/.gitignore b/.gitignore index 53eba52..12ab5bd 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ data.json # Exclude macOS Finder (System Explorer) View States .DS_Store -docs/.vitepress/dist -docs/.vitepress/cache -rebuild +docs/.vitepress/dist +docs/.vitepress/cache +rebuild +better_sqlite3.node diff --git a/README.md b/README.md index bb0348e..c0860a4 100644 --- a/README.md +++ b/README.md @@ -1,96 +1,2 @@ -# Obsidian Sample Plugin +# Obsidian SQLSeal -This is a sample plugin for Obsidian (https://obsidian.md). - -This project uses Typescript to provide type checking and documentation. -The repo depends on the latest plugin API (obsidian.d.ts) in Typescript Definition format, which contains TSDoc comments describing what it does. - -**Note:** The Obsidian API is still in early alpha and is subject to change at any time! - -This sample plugin demonstrates some of the basic functionality the plugin API can do. -- Adds a ribbon icon, which shows a Notice when clicked. -- Adds a command "Open Sample Modal" which opens a Modal. -- Adds a plugin setting tab to the settings page. -- Registers a global click event and output 'click' to the console. -- Registers a global interval which logs 'setInterval' to the console. - -## First time developing plugins? - -Quick starting guide for new plugin devs: - -- Check if [someone already developed a plugin for what you want](https://obsidian.md/plugins)! There might be an existing plugin similar enough that you can partner up with. -- Make a copy of this repo as a template with the "Use this template" button (login to GitHub if you don't see it). -- Clone your repo to a local development folder. For convenience, you can place this folder in your `.obsidian/plugins/your-plugin-name` folder. -- Install NodeJS, then run `npm i` in the command line under your repo folder. -- Run `npm run dev` to compile your plugin from `main.ts` to `main.js`. -- Make changes to `main.ts` (or create new `.ts` files). Those changes should be automatically compiled into `main.js`. -- Reload Obsidian to load the new version of your plugin. -- Enable plugin in settings window. -- For updates to the Obsidian API run `npm update` in the command line under your repo folder. - -## Releasing new releases - -- Update your `manifest.json` with your new version number, such as `1.0.1`, and the minimum Obsidian version required for your latest release. -- Update your `versions.json` file with `"new-plugin-version": "minimum-obsidian-version"` so older versions of Obsidian can download an older version of your plugin that's compatible. -- Create new GitHub release using your new version number as the "Tag version". Use the exact version number, don't include a prefix `v`. See here for an example: https://github.com/obsidianmd/obsidian-sample-plugin/releases -- Upload the files `manifest.json`, `main.js`, `styles.css` as binary attachments. Note: The manifest.json file must be in two places, first the root path of your repository and also in the release. -- Publish the release. - -> You can simplify the version bump process by running `npm version patch`, `npm version minor` or `npm version major` after updating `minAppVersion` manually in `manifest.json`. -> The command will bump version in `manifest.json` and `package.json`, and add the entry for the new version to `versions.json` - -## Adding your plugin to the community plugin list - -- Check https://github.com/obsidianmd/obsidian-releases/blob/master/plugin-review.md -- Publish an initial version. -- Make sure you have a `README.md` file in the root of your repo. -- Make a pull request at https://github.com/obsidianmd/obsidian-releases to add your plugin. - -## How to use - -- Clone this repo. -- Make sure your NodeJS is at least v16 (`node --version`). -- `npm i` or `yarn` to install dependencies. -- `npm run dev` to start compilation in watch mode. - -## Manually installing the plugin - -- Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`. - -## Improve code quality with eslint (optional) -- [ESLint](https://eslint.org/) is a tool that analyzes your code to quickly find problems. You can run ESLint against your plugin to find common bugs and ways to improve your code. -- To use eslint with this project, make sure to install eslint from terminal: - - `npm install -g eslint` -- To use eslint to analyze this project use this command: - - `eslint main.ts` - - eslint will then create a report with suggestions for code improvement by file and line number. -- If your source code is in a folder, such as `src`, you can use eslint with this command to analyze all files in that folder: - - `eslint .\src\` - -## Funding URL - -You can include funding URLs where people who use your plugin can financially support it. - -The simple way is to set the `fundingUrl` field to your link in your `manifest.json` file: - -```json -{ - "fundingUrl": "https://buymeacoffee.com" -} -``` - -If you have multiple URLs, you can also do: - -```json -{ - "fundingUrl": { - "Buy Me a Coffee": "https://buymeacoffee.com", - "GitHub Sponsor": "https://github.com/sponsors", - "Patreon": "https://www.patreon.com/" - } -} -``` - -## API Documentation - -See https://github.com/obsidianmd/obsidian-api diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 5b25e4d..35065bc 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -5,7 +5,7 @@ import replacePlugin from "esbuild-plugin-replace-regex"; const patchStr = ` const libPath = app.vault.adapter.getFullPath(app.vault.configDir) - addon = DEFAULT_ADDON || (DEFAULT_ADDON = require(path.resolve(libPath, "plugins/sqlseal/node_modules/better-sqlite3/build/Release/better_sqlite3.node"))); + addon = DEFAULT_ADDON || (DEFAULT_ADDON = require(path.resolve(libPath, "plugins/sqlseal/better_sqlite3.node"))); ` const banner = diff --git a/main.ts b/main.ts index cccaf98..fc9ff37 100644 --- a/main.ts +++ b/main.ts @@ -1,6 +1,5 @@ -import { MarkdownRenderer, MarkdownView, Plugin } from 'obsidian'; -import { SealFileSync } from 'src/SealFileSync'; -import { SqlSealSettings, SqlSealSettingsTab } from 'src/settings'; +import { Plugin } from 'obsidian'; +import { SqlSealSettings } from 'src/settings'; import { SqlSeal } from 'src/sqlSeal'; const DEFAULT_SETTINGS = { rows: [] } @@ -12,6 +11,8 @@ export default class SqlSealPlugin extends Plugin { await this.loadSettings(); const sqlSeal = new SqlSeal(this.app, false) this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler()) + await sqlSeal.connect() + // const fileSync = new SealFileSync(this.app, sqlSeal) // setTimeout(() => { diff --git a/package.json b/package.json index 4b23c8d..2651de4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,8 @@ + { - "name": "obsidian-sample-plugin", - "version": "1.0.0", - "description": "This is a sample plugin for Obsidian (https://obsidian.md)", + "name": "obsidian-sqlseal", + "version": "0.1.0", + "description": "A plugin for Obsidian that allows you to run SQL queries on your notes.", "main": "main.js", "scripts": { "dev": "node esbuild.config.mjs", @@ -37,6 +38,7 @@ "dependencies": { "better-sqlite3": "^9.5.0", "gray-matter": "^4.0.3", + "node-fetch": "^3.3.2", "node-sql-parser": "^5.0.0", "papaparse": "^5.4.1", "sqlite3": "^5.1.7" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3909bbd..12babb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ dependencies: gray-matter: specifier: ^4.0.3 version: 4.0.3 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 node-sql-parser: specifier: ^5.0.0 version: 5.1.0 @@ -1773,6 +1776,11 @@ packages: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} dev: true + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + /debug@4.3.4: resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'} @@ -2223,6 +2231,14 @@ packages: pend: 1.2.0 dev: true + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + dev: false + /file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -2268,6 +2284,13 @@ packages: tabbable: 6.2.0 dev: true + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + /fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} dev: false @@ -3039,6 +3062,20 @@ packages: semver: 7.6.0 dev: true + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + + /node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + /node-gyp-build@4.8.0: resolution: {integrity: sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==} hasBin: true @@ -3971,6 +4008,11 @@ packages: defaults: 1.0.4 dev: true + /web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + dev: false + /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} diff --git a/src/database.ts b/src/database.ts index c91efb4..f0af344 100644 --- a/src/database.ts +++ b/src/database.ts @@ -3,65 +3,108 @@ import { App } from "obsidian" import path from 'path' import Papa from 'papaparse' import { prefixedIfNotGlobal } from "./sqlReparseTables" +import fs from 'fs' +import { fetchBlobData } from "./utils" function isNumeric(str: string) { - if (typeof str != "string") return false // we only process strings! - return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)... - !isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail - } + if (typeof str != "string") return false // we only process strings! + return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)... + !isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail +} const toTypeStatements = (header: Array, data: Array>) => { - let d: Array> = data - const types: Record> = {} - header.forEach(key => { - const type = predictType(key, data) - if (type === 'REAL' || type === 'INTEGER') { - // converting all data here to text - d = d.map(record => ({ - ...record, - [key]: type === 'REAL' - ? parseFloat(record[key] as string) - : parseInt(record[key] as string) - })) - } + let d: Array> = data + const types: Record> = {} + header.forEach(key => { + const type = predictType(key, data) + if (type === 'REAL' || type === 'INTEGER') { + // converting all data here to text + d = d.map(record => ({ + ...record, + [key]: type === 'REAL' + ? parseFloat(record[key] as string) + : parseInt(record[key] as string) + })) + } - types[key] = type - }) + types[key] = type + }) - return { - data: d, - types - } + return { + data: d, + types + } } const predictType = (field: string, data: Array>) => { - if (field === 'id') { - return 'TEXT' - } + if (field === 'id') { + return 'TEXT' + } - const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field]), true) - if (canBeNumber) { + const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field]), true) + if (canBeNumber) { - // Check if Integer or Float - const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field]) === parseInt(d[field]), true) - if (canBeInteger) { - return 'INTEGER' - } + // Check if Integer or Float + const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field]) === parseInt(d[field]), true) + if (canBeInteger) { + return 'INTEGER' + } - return 'REAL' - } - return 'TEXT' + return 'REAL' + } + return 'TEXT' } +const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + export class SqlSealDatabase { private savedDatabases: Record = {} db: typeof Database - constructor(private readonly app: App, verbose = false) { + private isConnected = false + private connectingPromise; + private connectingPromiseResolve; + constructor(private readonly app: App, private readonly verbose = false) { + + } + + + async connect() { + if (this.isConnected) { + return Promise.resolve() + } + + if (this.connectingPromise) { + return this.connectingPromise + } + + this.connectingPromise = new Promise((resolve) => { + this.connectingPromiseResolve = resolve + }) //@ts-ignore - const defaultDbPath = path.resolve(app.vault.adapter.basePath,app.vault.configDir, "obsidian.db") - this.db = new Database(defaultDbPath, { verbose: verbose ? console.log : undefined }) + const dbPath = path.resolve(this.app.vault.adapter.basePath, this.app.vault.configDir, "plugins/sqlseal/better_sqlite3.node") + // check if file "better_sqlite3.node" exists in the plugin folder, if not, download it from github release + if (!fs.existsSync(dbPath)) { + console.log('BETTER SQL DOES NOT EXIST, DOWNLOADING FROM GITHUB RELEASE') + const url = 'https://github.com/kulak-at/tag-test/releases/download/0.1/better_sqlite3.node' // my github release url + + // use node native request to fetch data + console.log('PATH TO WRITE', dbPath) + + await fetchBlobData(url, dbPath) + } + console.log('FETCHED') + + await delay(1000) // Making sure everything is in order + + //@ts-ignore + const defaultDbPath = path.resolve(this.app.vault.adapter.basePath, this.app.vault.configDir, "obsidian.db") + this.db = new Database(defaultDbPath, { verbose: this.verbose ? console.log : undefined }) + + console.log('Connected to database', this.db) + this.isConnected = true + this.connectingPromiseResolve() } async createTableWithData(name: string, data: Array>) { @@ -100,7 +143,7 @@ export class SqlSealDatabase { data[field] = null } }) - insert.run(data) + insert.run(data) } catch (e) { console.log(e) } @@ -110,7 +153,7 @@ export class SqlSealDatabase { return insertMany(data) } - async createTable(name: string, fields: Record) { + async createTable(name: string, fields: Record) { const sqlFields = Object.entries(fields).map(([key, type]) => `${key} ${type}`) // FIXME: probably use schema generator, for now create with hardcoded fields await this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run() @@ -137,37 +180,37 @@ export class SqlSealDatabase { return } const file = this.app.vault.getFileByPath(url) - if (!file) { - console.log('File not found') - return - } - const data = await this.app.vault.cachedRead(file) - - const parsed = Papa.parse(data, { - header: true, - dynamicTyping: false, - skipEmptyLines: true - }) - const fields = parsed.meta.fields - const { data: parsedData, types } = toTypeStatements(fields, parsed.data) + if (!file) { + console.log('File not found') + return + } + const data = await this.app.vault.cachedRead(file) - await this.createTable(name, types) - // this.savedDatabases[name] = url - - // Purge the database - await this.db.prepare(`DELETE FROM ${name}`).run() - - const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); - const insertMany = this.db.transaction((pData: Array>) => { - pData.forEach(data => { - try { + const parsed = Papa.parse(data, { + header: true, + dynamicTyping: false, + skipEmptyLines: true + }) + const fields = parsed.meta.fields + const { data: parsedData, types } = toTypeStatements(fields, parsed.data) + + await this.createTable(name, types) + // this.savedDatabases[name] = url + + // Purge the database + await this.db.prepare(`DELETE FROM ${name}`).run() + + const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`); + const insertMany = this.db.transaction((pData: Array>) => { + pData.forEach(data => { + try { insert.run(data) - } catch (e) { - console.log(e) - } - }) + } catch (e) { + console.log(e) + } }) - - await insertMany(parsedData) + }) + + await insertMany(parsedData) } } \ No newline at end of file diff --git a/src/sqlSeal.ts b/src/sqlSeal.ts index 6043619..6a231a2 100644 --- a/src/sqlSeal.ts +++ b/src/sqlSeal.ts @@ -1,8 +1,7 @@ -import * as Database from "better-sqlite3"; import GrayMatter from "gray-matter"; -import { SqlSealDatabase, defineDatabaseFromUrl, instantiateDatabase } from "./database"; -import { displayData, displayError, displayInfo } from "./ui"; -import { App, MarkdownPostProcessorContext, MarkdownSectionInformation, WorkspaceLeaf } from "obsidian"; +import { SqlSealDatabase } from "./database"; +import { displayData, displayError, displayInfo, displayLoader } from "./ui"; +import { App, MarkdownPostProcessorContext } from "obsidian"; import { updateTables } from "./sqlReparseTables"; import { hashString } from "./hash"; @@ -12,6 +11,10 @@ export class SqlSeal { this.db = new SqlSealDatabase(app, verbose) } + async connect() { + await this.db.connect() + } + async resolveFrontmatter(ctx: MarkdownPostProcessorContext) { if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) { return ctx.frontmatter as Record @@ -29,6 +32,10 @@ export class SqlSeal { getHandler() { return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => { + + displayLoader(el) + await this.connect() + const frontmatter = await this.resolveFrontmatter(ctx) const prefix = hashString(ctx.sourcePath) const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g; diff --git a/src/ui.ts b/src/ui.ts index 6702ea4..a75268d 100644 --- a/src/ui.ts +++ b/src/ui.ts @@ -1,4 +1,5 @@ export const displayData = (el: HTMLElement, columns, data) => { + el.empty() const table = el.createEl("table") table.style.setProperty('width', '100%'); @@ -19,11 +20,19 @@ export const displayData = (el: HTMLElement, columns, data) => { } export const displayError = (el: HTMLElement, e: Error) => { + el.empty() const callout = el.createEl("div", { text: e.toString(), cls: 'callout' }) callout.dataset.callout = 'error' } export const displayInfo = (el: HTMLElement, message: string) => { + el.empty() + el.childNodes.forEach(c => c.remove()) const callout = el.createEl("div", { text: message, cls: 'callout' }) callout.dataset.callout = 'info' +} + +export const displayLoader = (el: HTMLElement) => { + const loader = el.createEl("div", { cls: 'callout', text: 'Loading SQLSeal database...' }) + loader.dataset.callout = 'info' } \ No newline at end of file diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..dfeb8d3 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,53 @@ + +import { get } from "https" +import * as fs from 'fs' + +export const fetchBlobData = async (url: string, filePath: string) => { + return new Promise((resolve, reject) => { + get(url, (response) => { + const statusCode = response.statusCode; + const contentType = response.headers['content-type']; + + let error; + if (statusCode === 301 || statusCode === 302) { + // Handle redirect + const redirectUrl = response.headers.location; + console.log(`Redirected to: ${redirectUrl}`); + if (!redirectUrl) { + reject('Redirect URL not found') + return; + } + fetchBlobData(redirectUrl, filePath).then(resolve).catch(reject); + return; + } + if (statusCode !== 200) { + error = new Error(`Request Failed. Status Code: ${statusCode}`); + } else if (!/^application\/octet-stream/.test(contentType ?? '')) { + error = new Error(`Invalid content-type. Expected application/octet-stream but received ${contentType}`); + } + if (error) { + console.error(error.message); + // Consume response data to free up memory + response.resume(); + return; + } + + const fileStream = fs.createWriteStream(filePath); + response.pipe(fileStream); + + fileStream.on('finish', () => { + console.log(`Blob data saved to ${filePath}`); + resolve() + fileStream.close() + }); + + fileStream.on('error', (err) => { + console.error(`Error writing to ${filePath}: ${err}`); + reject(`Error writing to ${filePath}: ${err}`) + }); + }).on('error', (err) => { + console.error(`Error fetching blob data: ${err.message}`); + reject(`Error writing to ${filePath}: ${err}`) + }); + }) + }; \ No newline at end of file