obsidian-plugin/h-sphere_sql-seal
1
0
Fork 0
mirror of https://github.com/h-sphere/sql-seal.git synced 2026-07-22 10:10:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

deploy script, publish script

Browse source
This commit is contained in:
Kacper Kula 2024-05-03 21:18:13 +01:00
parent 48639e8171
commit aae1a6c3e6
12 changed files with 283 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.github/workflows/deploy-docs.yml → .github/workflows/docs.yml vendored
View file

40
.github/workflows/publish.yml vendored Normal file
View file

@ -0,0 +1,40 @@
name: Release Obsidian plugin
on:
push:
tags:
- "*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Use Node.js
uses: actions/setup-node@v3
with:
node-version: "18.x"
- name: Build plugin
run: |
npm install
npm run build
- name: Recompile Better SQLite3 for Electron
run: |
npm run rebuild
- name: Copy better SQLite3 to plugin
run: |
cp -r node_modules/better-sqlite3/build/Release/better_sqlite3.node .
- name: Create release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag="${GITHUB_REF#refs/tags/}"
gh release create "$tag" \
--title="$tag" \
--draft \
main.js manifest.json styles.css better_sqlite3.node

7
.gitignore vendored
View file

@ -20,6 +20,7 @@ data.json
# Exclude macOS Finder (System Explorer) View States
.DS_Store
docs/.vitepress/dist
docs/.vitepress/cache
rebuild
docs/.vitepress/dist
docs/.vitepress/cache
rebuild
better_sqlite3.node

96
README.md
View file

@ -1,96 +1,2 @@
# Obsidian Sample Plugin
# Obsidian SQLSeal
This is a sample plugin for Obsidian (https://obsidian.md).
This project uses Typescript to provide type checking and documentation.
The repo depends on the latest plugin API (obsidian.d.ts) in Typescript Definition format, which contains TSDoc comments describing what it does.
**Note:** The Obsidian API is still in early alpha and is subject to change at any time!
This sample plugin demonstrates some of the basic functionality the plugin API can do.
- Adds a ribbon icon, which shows a Notice when clicked.
- Adds a command "Open Sample Modal" which opens a Modal.
- Adds a plugin setting tab to the settings page.
- Registers a global click event and output 'click' to the console.
- Registers a global interval which logs 'setInterval' to the console.
## First time developing plugins?
Quick starting guide for new plugin devs:
- Check if [someone already developed a plugin for what you want](https://obsidian.md/plugins)! There might be an existing plugin similar enough that you can partner up with.
- Make a copy of this repo as a template with the "Use this template" button (login to GitHub if you don't see it).
- Clone your repo to a local development folder. For convenience, you can place this folder in your `.obsidian/plugins/your-plugin-name` folder.
- Install NodeJS, then run `npm i` in the command line under your repo folder.
- Run `npm run dev` to compile your plugin from `main.ts` to `main.js`.
- Make changes to `main.ts` (or create new `.ts` files). Those changes should be automatically compiled into `main.js`.
- Reload Obsidian to load the new version of your plugin.
- Enable plugin in settings window.
- For updates to the Obsidian API run `npm update` in the command line under your repo folder.
## Releasing new releases
- Update your `manifest.json` with your new version number, such as `1.0.1`, and the minimum Obsidian version required for your latest release.
- Update your `versions.json` file with `"new-plugin-version": "minimum-obsidian-version"` so older versions of Obsidian can download an older version of your plugin that's compatible.
- Create new GitHub release using your new version number as the "Tag version". Use the exact version number, don't include a prefix `v`. See here for an example: https://github.com/obsidianmd/obsidian-sample-plugin/releases
- Upload the files `manifest.json`, `main.js`, `styles.css` as binary attachments. Note: The manifest.json file must be in two places, first the root path of your repository and also in the release.
- Publish the release.
> You can simplify the version bump process by running `npm version patch`, `npm version minor` or `npm version major` after updating `minAppVersion` manually in `manifest.json`.
> The command will bump version in `manifest.json` and `package.json`, and add the entry for the new version to `versions.json`
## Adding your plugin to the community plugin list
- Check https://github.com/obsidianmd/obsidian-releases/blob/master/plugin-review.md
- Publish an initial version.
- Make sure you have a `README.md` file in the root of your repo.
- Make a pull request at https://github.com/obsidianmd/obsidian-releases to add your plugin.
## How to use
- Clone this repo.
- Make sure your NodeJS is at least v16 (`node --version`).
- `npm i` or `yarn` to install dependencies.
- `npm run dev` to start compilation in watch mode.
## Manually installing the plugin
- Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`.
## Improve code quality with eslint (optional)
- [ESLint](https://eslint.org/) is a tool that analyzes your code to quickly find problems. You can run ESLint against your plugin to find common bugs and ways to improve your code.
- To use eslint with this project, make sure to install eslint from terminal:
- `npm install -g eslint`
- To use eslint to analyze this project use this command:
- `eslint main.ts`
- eslint will then create a report with suggestions for code improvement by file and line number.
- If your source code is in a folder, such as `src`, you can use eslint with this command to analyze all files in that folder:
- `eslint .\src\`
## Funding URL
You can include funding URLs where people who use your plugin can financially support it.
The simple way is to set the `fundingUrl` field to your link in your `manifest.json` file:
```json
{
"fundingUrl": "https://buymeacoffee.com"
}
```
If you have multiple URLs, you can also do:
```json
{
"fundingUrl": {
"Buy Me a Coffee": "https://buymeacoffee.com",
"GitHub Sponsor": "https://github.com/sponsors",
"Patreon": "https://www.patreon.com/"
}
}
```
## API Documentation
See https://github.com/obsidianmd/obsidian-api

2
esbuild.config.mjs
View file

@ -5,7 +5,7 @@ import replacePlugin from "esbuild-plugin-replace-regex";
const patchStr = `
const libPath = app.vault.adapter.getFullPath(app.vault.configDir)
addon = DEFAULT_ADDON || (DEFAULT_ADDON = require(path.resolve(libPath, "plugins/sqlseal/node_modules/better-sqlite3/build/Release/better_sqlite3.node")));
addon = DEFAULT_ADDON || (DEFAULT_ADDON = require(path.resolve(libPath, "plugins/sqlseal/better_sqlite3.node")));
`
const banner =

7
main.ts
View file

@ -1,6 +1,5 @@
import { MarkdownRenderer, MarkdownView, Plugin } from 'obsidian';
import { SealFileSync } from 'src/SealFileSync';
import { SqlSealSettings, SqlSealSettingsTab } from 'src/settings';
import { Plugin } from 'obsidian';
import { SqlSealSettings } from 'src/settings';
import { SqlSeal } from 'src/sqlSeal';
const DEFAULT_SETTINGS = { rows: [] }
@ -12,6 +11,8 @@ export default class SqlSealPlugin extends Plugin {
await this.loadSettings();
const sqlSeal = new SqlSeal(this.app, false)
this.registerMarkdownCodeBlockProcessor("sqlseal", sqlSeal.getHandler())
await sqlSeal.connect()
// const fileSync = new SealFileSync(this.app, sqlSeal)
// setTimeout(() => {

8
package.json
View file

@ -1,7 +1,8 @@
{
"name": "obsidian-sample-plugin",
"version": "1.0.0",
"description": "This is a sample plugin for Obsidian (https://obsidian.md)",
"name": "obsidian-sqlseal",
"version": "0.1.0",
"description": "A plugin for Obsidian that allows you to run SQL queries on your notes.",
"main": "main.js",
"scripts": {
"dev": "node esbuild.config.mjs",
@ -37,6 +38,7 @@
"dependencies": {
"better-sqlite3": "^9.5.0",
"gray-matter": "^4.0.3",
"node-fetch": "^3.3.2",
"node-sql-parser": "^5.0.0",
"papaparse": "^5.4.1",
"sqlite3": "^5.1.7"

42
pnpm-lock.yaml
View file

@ -11,6 +11,9 @@ dependencies:
gray-matter:
specifier: ^4.0.3
version: 4.0.3
node-fetch:
specifier: ^3.3.2
version: 3.3.2
node-sql-parser:
specifier: ^5.0.0
version: 5.1.0
@ -1773,6 +1776,11 @@ packages:
resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
dev: true
/data-uri-to-buffer@4.0.1:
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
engines: {node: '>= 12'}
dev: false
/debug@4.3.4:
resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
engines: {node: '>=6.0'}
@ -2223,6 +2231,14 @@ packages:
pend: 1.2.0
dev: true
/fetch-blob@3.2.0:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
engines: {node: ^12.20 || >= 14.13}
dependencies:
node-domexception: 1.0.0
web-streams-polyfill: 3.3.3
dev: false
/file-entry-cache@6.0.1:
resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==}
engines: {node: ^10.12.0 || >=12.0.0}
@ -2268,6 +2284,13 @@ packages:
tabbable: 6.2.0
dev: true
/formdata-polyfill@4.0.10:
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
dependencies:
fetch-blob: 3.2.0
dev: false
/fs-constants@1.0.0:
resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
dev: false
@ -3039,6 +3062,20 @@ packages:
semver: 7.6.0
dev: true
/node-domexception@1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
dev: false
/node-fetch@3.3.2:
resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dependencies:
data-uri-to-buffer: 4.0.1
fetch-blob: 3.2.0
formdata-polyfill: 4.0.10
dev: false
/node-gyp-build@4.8.0:
resolution: {integrity: sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==}
hasBin: true
@ -3971,6 +4008,11 @@ packages:
defaults: 1.0.4
dev: true
/web-streams-polyfill@3.3.3:
resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
engines: {node: '>= 8'}
dev: false
/which@2.0.2:
resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
engines: {node: '>= 8'}

183
src/database.ts
View file

@ -3,65 +3,108 @@ import { App } from "obsidian"
import path from 'path'
import Papa from 'papaparse'
import { prefixedIfNotGlobal } from "./sqlReparseTables"
import fs from 'fs'
import { fetchBlobData } from "./utils"
function isNumeric(str: string) {
if (typeof str != "string") return false // we only process strings!
return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
if (typeof str != "string") return false // we only process strings!
return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
const toTypeStatements = (header: Array<string>, data: Array<Record<string, string>>) => {
let d: Array<Record<string, string|number>> = data
const types: Record<string, ReturnType<typeof predictType>> = {}
header.forEach(key => {
const type = predictType(key, data)
if (type === 'REAL' || type === 'INTEGER') {
// converting all data here to text
d = d.map(record => ({
...record,
[key]: type === 'REAL'
? parseFloat(record[key] as string)
: parseInt(record[key] as string)
}))
}
let d: Array<Record<string, string | number>> = data
const types: Record<string, ReturnType<typeof predictType>> = {}
header.forEach(key => {
const type = predictType(key, data)
if (type === 'REAL' || type === 'INTEGER') {
// converting all data here to text
d = d.map(record => ({
...record,
[key]: type === 'REAL'
? parseFloat(record[key] as string)
: parseInt(record[key] as string)
}))
}
types[key] = type
})
types[key] = type
})
return {
data: d,
types
}
return {
data: d,
types
}
}
const predictType = (field: string, data: Array<Record<string, string>>) => {
if (field === 'id') {
return 'TEXT'
}
if (field === 'id') {
return 'TEXT'
}
const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field]), true)
if (canBeNumber) {
const canBeNumber = data.reduce((acc, d) => acc && isNumeric(d[field]), true)
if (canBeNumber) {
// Check if Integer or Float
const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field]) === parseInt(d[field]), true)
if (canBeInteger) {
return 'INTEGER'
}
// Check if Integer or Float
const canBeInteger = data.reduce((acc, d) => acc && parseFloat(d[field]) === parseInt(d[field]), true)
if (canBeInteger) {
return 'INTEGER'
}
return 'REAL'
}
return 'TEXT'
return 'REAL'
}
return 'TEXT'
}
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
export class SqlSealDatabase {
private savedDatabases: Record<string, any> = {}
db: typeof Database
constructor(private readonly app: App, verbose = false) {
private isConnected = false
private connectingPromise;
private connectingPromiseResolve;
constructor(private readonly app: App, private readonly verbose = false) {
}
async connect() {
if (this.isConnected) {
return Promise.resolve()
}
if (this.connectingPromise) {
return this.connectingPromise
}
this.connectingPromise = new Promise((resolve) => {
this.connectingPromiseResolve = resolve
})
//@ts-ignore
const defaultDbPath = path.resolve(app.vault.adapter.basePath,app.vault.configDir, "obsidian.db")
this.db = new Database(defaultDbPath, { verbose: verbose ? console.log : undefined })
const dbPath = path.resolve(this.app.vault.adapter.basePath, this.app.vault.configDir, "plugins/sqlseal/better_sqlite3.node")
// check if file "better_sqlite3.node" exists in the plugin folder, if not, download it from github release
if (!fs.existsSync(dbPath)) {
console.log('BETTER SQL DOES NOT EXIST, DOWNLOADING FROM GITHUB RELEASE')
const url = 'https://github.com/kulak-at/tag-test/releases/download/0.1/better_sqlite3.node' // my github release url
// use node native request to fetch data
console.log('PATH TO WRITE', dbPath)
await fetchBlobData(url, dbPath)
}
console.log('FETCHED')
await delay(1000) // Making sure everything is in order
//@ts-ignore
const defaultDbPath = path.resolve(this.app.vault.adapter.basePath, this.app.vault.configDir, "obsidian.db")
this.db = new Database(defaultDbPath, { verbose: this.verbose ? console.log : undefined })
console.log('Connected to database', this.db)
this.isConnected = true
this.connectingPromiseResolve()
}
async createTableWithData(name: string, data: Array<Record<string, unknown>>) {
@ -100,7 +143,7 @@ export class SqlSealDatabase {
data[field] = null
}
})
insert.run(data)
insert.run(data)
} catch (e) {
console.log(e)
}
@ -110,7 +153,7 @@ export class SqlSealDatabase {
return insertMany(data)
}
async createTable(name: string, fields: Record<string, 'TEXT'|'INTEGER'|'REAL'>) {
async createTable(name: string, fields: Record<string, 'TEXT' | 'INTEGER' | 'REAL'>) {
const sqlFields = Object.entries(fields).map(([key, type]) => `${key} ${type}`)
// FIXME: probably use schema generator, for now create with hardcoded fields
await this.db.prepare(`DROP TABLE IF EXISTS ${name}`).run()
@ -137,37 +180,37 @@ export class SqlSealDatabase {
return
}
const file = this.app.vault.getFileByPath(url)
if (!file) {
console.log('File not found')
return
}
const data = await this.app.vault.cachedRead(file)
const parsed = Papa.parse(data, {
header: true,
dynamicTyping: false,
skipEmptyLines: true
})
const fields = parsed.meta.fields
const { data: parsedData, types } = toTypeStatements(fields, parsed.data)
if (!file) {
console.log('File not found')
return
}
const data = await this.app.vault.cachedRead(file)
await this.createTable(name, types)
// this.savedDatabases[name] = url
// Purge the database
await this.db.prepare(`DELETE FROM ${name}`).run()
const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`);
const insertMany = this.db.transaction((pData: Array<Record<string, any>>) => {
pData.forEach(data => {
try {
const parsed = Papa.parse(data, {
header: true,
dynamicTyping: false,
skipEmptyLines: true
})
const fields = parsed.meta.fields
const { data: parsedData, types } = toTypeStatements(fields, parsed.data)
await this.createTable(name, types)
// this.savedDatabases[name] = url
// Purge the database
await this.db.prepare(`DELETE FROM ${name}`).run()
const insert = this.db.prepare(`INSERT INTO ${name} (${fields.join(', ')}) VALUES (${fields.map((key: string) => '@' + key).join(', ')})`);
const insertMany = this.db.transaction((pData: Array<Record<string, any>>) => {
pData.forEach(data => {
try {
insert.run(data)
} catch (e) {
console.log(e)
}
})
} catch (e) {
console.log(e)
}
})
await insertMany(parsedData)
})
await insertMany(parsedData)
}
}

15
src/sqlSeal.ts
View file

@ -1,8 +1,7 @@
import * as Database from "better-sqlite3";
import GrayMatter from "gray-matter";
import { SqlSealDatabase, defineDatabaseFromUrl, instantiateDatabase } from "./database";
import { displayData, displayError, displayInfo } from "./ui";
import { App, MarkdownPostProcessorContext, MarkdownSectionInformation, WorkspaceLeaf } from "obsidian";
import { SqlSealDatabase } from "./database";
import { displayData, displayError, displayInfo, displayLoader } from "./ui";
import { App, MarkdownPostProcessorContext } from "obsidian";
import { updateTables } from "./sqlReparseTables";
import { hashString } from "./hash";
@ -12,6 +11,10 @@ export class SqlSeal {
this.db = new SqlSealDatabase(app, verbose)
}
async connect() {
await this.db.connect()
}
async resolveFrontmatter(ctx: MarkdownPostProcessorContext) {
if (ctx.frontmatter && Object.keys(ctx.frontmatter).length > 0) {
return ctx.frontmatter as Record<string, any>
@ -29,6 +32,10 @@ export class SqlSeal {
getHandler() {
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
displayLoader(el)
await this.connect()
const frontmatter = await this.resolveFrontmatter(ctx)
const prefix = hashString(ctx.sourcePath)
const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g;

9
src/ui.ts
View file

@ -1,4 +1,5 @@
export const displayData = (el: HTMLElement, columns, data) => {
el.empty()
const table = el.createEl("table")
table.style.setProperty('width', '100%');
@ -19,11 +20,19 @@ export const displayData = (el: HTMLElement, columns, data) => {
}
export const displayError = (el: HTMLElement, e: Error) => {
el.empty()
const callout = el.createEl("div", { text: e.toString(), cls: 'callout' })
callout.dataset.callout = 'error'
}
export const displayInfo = (el: HTMLElement, message: string) => {
el.empty()
el.childNodes.forEach(c => c.remove())
const callout = el.createEl("div", { text: message, cls: 'callout' })
callout.dataset.callout = 'info'
}
export const displayLoader = (el: HTMLElement) => {
const loader = el.createEl("div", { cls: 'callout', text: 'Loading SQLSeal database...' })
loader.dataset.callout = 'info'
}

53
src/utils.ts Normal file
View file

@ -0,0 +1,53 @@
import { get } from "https"
import * as fs from 'fs'
export const fetchBlobData = async (url: string, filePath: string) => {
return new Promise<void>((resolve, reject) => {
get(url, (response) => {
const statusCode = response.statusCode;
const contentType = response.headers['content-type'];
let error;
if (statusCode === 301 || statusCode === 302) {
// Handle redirect
const redirectUrl = response.headers.location;
console.log(`Redirected to: ${redirectUrl}`);
if (!redirectUrl) {
reject('Redirect URL not found')
return;
}
fetchBlobData(redirectUrl, filePath).then(resolve).catch(reject);
return;
}
if (statusCode !== 200) {
error = new Error(`Request Failed. Status Code: ${statusCode}`);
} else if (!/^application\/octet-stream/.test(contentType ?? '')) {
error = new Error(`Invalid content-type. Expected application/octet-stream but received ${contentType}`);
}
if (error) {
console.error(error.message);
// Consume response data to free up memory
response.resume();
return;
}
const fileStream = fs.createWriteStream(filePath);
response.pipe(fileStream);
fileStream.on('finish', () => {
console.log(`Blob data saved to ${filePath}`);
resolve()
fileStream.close()
});
fileStream.on('error', (err) => {
console.error(`Error writing to ${filePath}: ${err}`);
reject(`Error writing to ${filePath}: ${err}`)
});
}).on('error', (err) => {
console.error(`Error fetching blob data: ${err.message}`);
reject(`Error writing to ${filePath}: ${err}`)
});
})
};