feat: adding proper gramar parsing instead of regex. Improving how CSVs are handled and reloaded.

This commit is contained in:
Kacper Kula 2024-09-02 19:40:48 +01:00
parent 645c74daa3
commit 8a5c2bbad5
23 changed files with 2061 additions and 130 deletions

1
.gitignore vendored
View file

@ -27,3 +27,4 @@ better_sqlite3.node
better_sqlite3*.node
database.sqlite
coverage/

47
SqlSealLang.g4 Normal file
View file

@ -0,0 +1,47 @@
grammar SqlSealLang;
parse
: statement+ EOF
;
statement
: tableStatement
| queryStatement
;
tableStatement
: TABLE WS+ ID WS* '=' WS* FILE WS* '(' WS* FILE_URL WS* ')' WS*
;
queryStatement
: withClause? selectStatement WS*
;
withClause
: WITH WS+ cteDefinition (WS* ',' WS* cteDefinition)* WS*
;
cteDefinition
: ID WS+ AS WS* '(' WS* selectStatement WS* ')'
;
selectStatement
: SELECT WS+ selectBody
;
selectBody
: .*?
;
TABLE : 'TABLE';
FILE : 'file';
WITH : 'WITH';
AS : 'AS';
SELECT : 'SELECT';
ID : [a-zA-Z_][a-zA-Z_0-9]*;
FILE_URL : [a-zA-Z0-9_./\\-]+;
WS : [ \t\r\n]+;
// Catch-all rule for any other character
ANY : .;

67
package-lock.json generated
View file

@ -1,27 +1,32 @@
{
"name": "sqlseal",
"version": "0.4.1",
"version": "0.6.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "sqlseal",
"version": "0.4.1",
"version": "0.6.0",
"license": "MIT",
"dependencies": {
"antlr4": "^4.13.2",
"antlr4ts": "0.5.0-alpha.4",
"better-sqlite3": "^11.2.1",
"json5": "^2.2.3",
"lodash": "^4.17.21",
"node-sql-parser": "^5.0.0",
"papaparse": "^5.4.1"
},
"devDependencies": {
"@jest/globals": "^29.7.0",
"@types/better-sqlite3": "^7.6.10",
"@types/jest": "^29.5.12",
"@types/lodash": "^4.17.7",
"@types/node": "^16.11.6",
"@types/node": "^22.5.0",
"@types/papaparse": "^5.3.14",
"@typescript-eslint/eslint-plugin": "5.29.0",
"@typescript-eslint/parser": "5.29.0",
"antlr4ts-cli": "0.5.0-alpha.4",
"builtin-modules": "3.3.0",
"electron": "^30.0.0",
"electron-rebuild": "^3.2.9",
@ -2478,10 +2483,21 @@
"dev": true
},
"node_modules/@types/node": {
"version": "16.18.96",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.96.tgz",
"integrity": "sha512-84iSqGXoO+Ha16j8pRZ/L90vDMKX04QTYMTfYeE1WrjWaZXuchBehGUZEpNgx7JnmlrIHdnABmpjrQjhCnNldQ==",
"dev": true
"version": "22.5.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.2.tgz",
"integrity": "sha512-acJsPTEqYqulZS/Yp/S3GgeE6GZ0qYODUR8aVr/DkhHQ8l9nd4j5x1/ZJy9/gHrRlFMqkO6i0I3E27Alu4jjPg==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.19.2"
}
},
"node_modules/@types/node/node_modules/undici-types": {
"version": "6.19.8",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
"integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/papaparse": {
"version": "5.3.14",
@ -3227,6 +3243,31 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/antlr4": {
"version": "4.13.2",
"resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz",
"integrity": "sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=16"
}
},
"node_modules/antlr4ts": {
"version": "0.5.0-alpha.4",
"resolved": "https://registry.npmjs.org/antlr4ts/-/antlr4ts-0.5.0-alpha.4.tgz",
"integrity": "sha512-WPQDt1B74OfPv/IMS2ekXAKkTZIHl88uMetg6q3OTqgFxZ/dxDXI0EWLyZid/1Pe6hTftyg5N7gel5wNAGxXyQ==",
"license": "BSD-3-Clause"
},
"node_modules/antlr4ts-cli": {
"version": "0.5.0-alpha.4",
"resolved": "https://registry.npmjs.org/antlr4ts-cli/-/antlr4ts-cli-0.5.0-alpha.4.tgz",
"integrity": "sha512-lVPVBTA2CVHRYILSKilL6Jd4hAumhSZZWA7UbQNQrmaSSj7dPmmYaN4bOmZG79cOy0lS00i4LY68JZZjZMWVrw==",
"dev": true,
"license": "BSD-3-Clause",
"bin": {
"antlr4ts": "antlr4ts"
}
},
"node_modules/anymatch": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
@ -6458,7 +6499,6 @@
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dev": true,
"license": "MIT",
"bin": {
"json5": "lib/cli.js"
@ -9131,17 +9171,6 @@
"node": ">=12"
}
},
"node_modules/vitepress/node_modules/@types/node": {
"version": "20.12.7",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz",
"integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==",
"dev": true,
"optional": true,
"peer": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/vitepress/node_modules/@vitejs/plugin-vue": {
"version": "5.0.4",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.0.4.tgz",

View file

@ -7,6 +7,7 @@
"dev": "node esbuild.config.mjs",
"typecheck": "tsc -noEmit -skipLibCheck",
"build": " node esbuild.config.mjs production",
"build-grammar": "antlr4ts -o src/grammar -Dlanguage=TypeScript -visitor SqlSealLang.g4",
"version": "node version-bump.mjs && git add manifest.json versions.json",
"rebuild": "electron-rebuild -f -w better-sqlite3",
"docs:dev": "vitepress dev docs",
@ -26,6 +27,8 @@
"@types/papaparse": "^5.3.14",
"@typescript-eslint/eslint-plugin": "5.29.0",
"@typescript-eslint/parser": "5.29.0",
"antlr4ng-cli": "^2.0.0",
"antlr4ts-cli": "0.5.0-alpha.4",
"builtin-modules": "3.3.0",
"electron": "^30.0.0",
"electron-rebuild": "^3.2.9",
@ -42,6 +45,10 @@
"vue": "^3.4.26"
},
"dependencies": {
"antlr4": "^4.13.2",
"antlr4-cli": "^4.5.3",
"antlr4ng": "^3.0.4",
"antlr4ts": "0.5.0-alpha.4",
"better-sqlite3": "^11.2.1",
"json5": "^2.2.3",
"lodash": "^4.17.21",

View file

@ -8,6 +8,18 @@ importers:
.:
dependencies:
antlr4:
specifier: ^4.13.2
version: 4.13.2
antlr4-cli:
specifier: ^4.5.3
version: 4.5.3
antlr4ng:
specifier: ^3.0.4
version: 3.0.4(antlr4ng-cli@2.0.0)
antlr4ts:
specifier: 0.5.0-alpha.4
version: 0.5.0-alpha.4
better-sqlite3:
specifier: ^11.2.1
version: 11.2.1
@ -48,6 +60,12 @@ importers:
'@typescript-eslint/parser':
specifier: 5.29.0
version: 5.29.0(eslint@8.57.0)(typescript@4.7.4)
antlr4ng-cli:
specifier: ^2.0.0
version: 2.0.0
antlr4ts-cli:
specifier: 0.5.0-alpha.4
version: 0.5.0-alpha.4
builtin-modules:
specifier: 3.3.0
version: 3.3.0
@ -1201,6 +1219,29 @@ packages:
resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==}
engines: {node: '>=10'}
antlr4-cli@4.5.3:
resolution: {integrity: sha512-2My0geojGhigXgy9J7/YGYXau8nVjUldxvwttsG/amYtUjTSfc149AQBeBtDsuleApY3C8oQq1RW+8RIum1uEQ==}
antlr4@4.13.2:
resolution: {integrity: sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==}
engines: {node: '>=16'}
antlr4ng-cli@2.0.0:
resolution: {integrity: sha512-oAt5OSSYhRQn1PgahtpAP4Vp3BApCoCqlzX7Q8ZUWWls4hX59ryYuu0t7Hwrnfk796OxP/vgIJaqxdltd/oEvQ==}
hasBin: true
antlr4ng@3.0.4:
resolution: {integrity: sha512-u1Ww6wVv9hq70E9AaYe5qW3ba8hvnjJdO3ZsKnb3iJWFV/medLEEhbyWwXCvvD2ef0ptdaiIUgmaazS/WE6uyQ==}
peerDependencies:
antlr4ng-cli: ^2.0.0
antlr4ts-cli@0.5.0-alpha.4:
resolution: {integrity: sha512-lVPVBTA2CVHRYILSKilL6Jd4hAumhSZZWA7UbQNQrmaSSj7dPmmYaN4bOmZG79cOy0lS00i4LY68JZZjZMWVrw==}
hasBin: true
antlr4ts@0.5.0-alpha.4:
resolution: {integrity: sha512-WPQDt1B74OfPv/IMS2ekXAKkTZIHl88uMetg6q3OTqgFxZ/dxDXI0EWLyZid/1Pe6hTftyg5N7gel5wNAGxXyQ==}
anymatch@3.1.3:
resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
engines: {node: '>= 8'}
@ -1941,6 +1982,10 @@ packages:
ini@1.3.8:
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
interpret@1.4.0:
resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==}
engines: {node: '>= 0.10'}
ip-address@9.0.5:
resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==}
engines: {node: '>= 12'}
@ -2614,6 +2659,10 @@ packages:
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
engines: {node: '>= 6'}
rechoir@0.6.2:
resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==}
engines: {node: '>= 0.10'}
regexpp@3.2.0:
resolution: {integrity: sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==}
engines: {node: '>=8'}
@ -2716,6 +2765,11 @@ packages:
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
engines: {node: '>=8'}
shelljs@0.7.8:
resolution: {integrity: sha512-/YF5Uk8hcwi7ima04ppkbA4RaRMdPMBfwAvAf8sufYOxsJRtbdoBsT8vGvlb+799BrlGdYrd+oczIA2eN2JdWA==}
engines: {node: '>=0.11.0'}
hasBin: true
shiki@1.14.1:
resolution: {integrity: sha512-FujAN40NEejeXdzPt+3sZ3F2dx1U24BY2XTY01+MG8mbxCiA2XukXdcbyMyLAHJ/1AUUnQd1tZlvIjefWWEJeA==}
@ -4281,6 +4335,22 @@ snapshots:
ansi-styles@5.2.0: {}
antlr4-cli@4.5.3:
dependencies:
shelljs: 0.7.8
antlr4@4.13.2: {}
antlr4ng-cli@2.0.0: {}
antlr4ng@3.0.4(antlr4ng-cli@2.0.0):
dependencies:
antlr4ng-cli: 2.0.0
antlr4ts-cli@0.5.0-alpha.4: {}
antlr4ts@0.5.0-alpha.4: {}
anymatch@3.1.3:
dependencies:
normalize-path: 3.0.0
@ -5157,6 +5227,8 @@ snapshots:
ini@1.3.8: {}
interpret@1.4.0: {}
ip-address@9.0.5:
dependencies:
jsbn: 1.1.0
@ -6027,6 +6099,10 @@ snapshots:
string_decoder: 1.3.0
util-deprecate: 1.0.2
rechoir@0.6.2:
dependencies:
resolve: 1.22.8
regexpp@3.2.0: {}
require-directory@2.1.1: {}
@ -6131,6 +6207,12 @@ snapshots:
shebang-regex@3.0.0: {}
shelljs@0.7.8:
dependencies:
glob: 7.2.3
interpret: 1.4.0
rechoir: 0.6.2
shiki@1.14.1:
dependencies:
'@shikijs/core': 1.14.1

View file

@ -40,6 +40,12 @@ export class SealObserver {
});
}
hasAnyObserver(tableNames: string[]) {
return tableNames.reduce((acc, t) =>
acc || (this.tables.has(t) && this.tables.get(t)!.size > 0),
false)
}
unregisterObserversByTag(tag: string) {
if (this.tags.has(tag)) {
const observers = this.tags.get(tag);

View file

@ -6,80 +6,80 @@ import { prefixedIfNotGlobal, updateTables } from "./sqlReparseTables"
import { SealObserver } from "./SealObserver"
import { SqlSealDatabase } from "./database"
import { Logger } from "./logger"
import { ParsedLanguage, parseLanguage, TableStatement } from "./grammar/parser"
import { SyncModel } from "./models/sync"
export class SqlSealCodeblockHandler {
get globalTables() {
return ['files', 'tags']
}
constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) { }
syncModel: SyncModel
constructor(private readonly app: App, private readonly db: SqlSealDatabase,private readonly observer: SealObserver, private logger: Logger) {
this.syncModel = new SyncModel(db)
}
tablesConfig: Record<string, string> = {}
setupTables(tables: ParsedLanguage['tables'], ctx: MarkdownPostProcessorContext) {
tables.forEach(table => {
this.setupTable(table, ctx)
})
}
getHandler() {
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
displayLoader(el)
await this.db.connect()
this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers.
const frontmatter = await resolveFrontmatter(ctx, this.app)
const prefix = hashString(ctx.sourcePath)
const regex = /TABLE\s+(.+)\s+=\s+file\(([^)]+)\)/g;
let match
while ((match = regex.exec(source)) !== null) {
const name = match[1];
const url = match[2];
this.logger.log('CodedblockHandler. table', name, url)
setupTable({ url, name }: TableStatement, ctx: MarkdownPostProcessorContext) {
// UPDATING TABLES IF THEY CHANGED SINCE LAST REGISTER
const prefix = hashString(ctx.sourcePath)
const prefixedName = prefixedIfNotGlobal(name, this.globalTables, prefix)
if (this.tablesConfig[prefixedName] === url) {
// We do not need to rework it. Also, it should be watched, right?
// continue
}
if (!!this.tablesConfig[prefixedName]) {
// We need to remove old database as it's pointing to the wrong collection (probably).
}
const existing = this.syncModel.getSync(ctx.sourcePath, prefixedName)
// FIXME: do not register observer when it's already been registed for this ctx.
this.observer.registerObserver(`file:${url}`, async () => {
// Update table
await this.db.loadDataForDatabaseFromUrl(prefixedName, url, true)
// update when it was updated.
this.syncModel.removeSync(ctx.sourcePath, prefixedName)
this.syncModel.registerSync(ctx.sourcePath, url, prefixedName)
// Fire observers for the table
this.observer.fireObservers(`table:${prefixedName}`)
}, ctx.docId)
if (this.tablesConfig[prefixedName] !== url) {
this.observer.fireObservers(`file:${url}`)
this.tablesConfig[prefixedName] = url
} else {
// We still need to update the table watchers?
requestAnimationFrame(() => {
this.observer.fireObservers(`table:${prefixedName}`)
})
if (existing) {
if (existing.url === url) {
// Already synced
// TODO: Check if the file was updated after we synced, if so, we should update it.
requestAnimationFrame(() => {
this.observer.fireObservers(`table:${prefixedName}`)
})
return
} else {
// Old database is no-no. We need to update it.
this.db.db.prepare('DROP TABLE :tablename').run({
tablename: prefixedName
})
this.syncModel.removeSync(ctx.sourcePath, prefixedName)
}
}
this.observer.fireObservers(`file:${url}`)
}
const selectRegexp = /SELECT\s+(.*)/g; // OR WITH?
const selectMatch = selectRegexp.exec(source)
if (selectMatch) {
setupSelect(selectStmt: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) {
try {
// FIXME: WAIT FOR TABLES TO BE READDY!!!!
const selectStatement = selectMatch[0]
const { statement, tables } = updateTables(selectStatement, this.globalTables, prefix)
const prefix = hashString(ctx.sourcePath)
const { statement, tables } = updateTables(selectStmt, this.globalTables, prefix)
const frontmatter = resolveFrontmatter(ctx, this.app)
const renderSelect = async () => {
try {
const stmt = await this.db.db.prepare(statement)
const columns = await stmt.columns().map(column => column.name);
const data = await stmt.all(frontmatter ?? {})
const stmt = this.db.db.prepare(statement)
const columns = stmt.columns().map(column => column.name);
const data = stmt.all(frontmatter ?? {})
displayData(el, columns, data)
} catch (e) {
displayError(el, e)
}
}
// Register observer for each table
tables.forEach(table => {
const observer = async () => {
@ -88,18 +88,22 @@ export class SqlSealCodeblockHandler {
this.observer.unregisterObserversByTag(ctx.docId)
}
displayLoader(el)
await renderSelect()
}
this.observer.registerObserver(`table:${table}`, observer, ctx.docId)
})
this.globalTables.forEach(t => this.observer.fireObservers(`table:${t}`))
// await renderSelect() <- this should get triggered automatically.
if (!this.observer.hasAnyObserver(this.globalTables.map(t => `table:${t}`))) {
renderSelect()
}
// Triggering the ones which exist
tables.forEach(t => {
if (this.syncModel.getSync(ctx.sourcePath, t)) {
this.observer.fireObservers(`table:${t}`)
}
})
} catch (e) {
if (e instanceof RangeError && ctx && Object.keys(ctx.frontmatter).length === 0) {
displayInfo(el, 'Cannot access frontmatter properties in Live Preview Mode. Switch to Reading Mode to see the results.')
@ -108,6 +112,27 @@ export class SqlSealCodeblockHandler {
}
}
}
getHandler() {
return async (source: string, el: HTMLElement, ctx: MarkdownPostProcessorContext) => {
displayLoader(el)
await this.db.connect()
this.observer.unregisterObserversByTag(ctx.docId) // Unregister all previous observers.
// NEW PARSING
try {
const results = parseLanguage(source)
// FIXME: if not properly parsed, we should display error.
this.setupTables(results.tables, ctx)
results.queryPart ? this.setupSelect(results.queryPart, el, ctx) : displayInfo(el, `SQLSeal. ${results.tables.map(t => `${t.name} table definition.`).join(' ')}`)
} catch (e) {
displayError(el, e.toString())
}
}
}
}

View file

@ -4,48 +4,10 @@ import path from 'path'
import Papa from 'papaparse'
import { prefixedIfNotGlobal } from "./sqlReparseTables"
import { camelCase } from 'lodash'
import { fetchBlobData, FieldTypes, predictJson, predictType } from "./utils"
import { dataToCamelCase, fetchBlobData, FieldTypes, predictJson, predictType, toTypeStatements } from "./utils"
import os from 'os'
import fs from 'fs'
export function isNumeric(str: string) {
if (typeof str != "string") return false // we only process strings!
return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
function dataToCamelCase(data: Array<Record<string, unknown>>) {
return data.map(d => Object.keys(d).reduce((acc, k) => ({
...acc,
[camelCase(k)]: d[k]
}), {}))
}
const toTypeStatements = (header: Array<string>, data: Array<Record<string, string>>) => {
let d: Array<Record<string, string | number>> = data
const types: Record<string, ReturnType<typeof predictType>> = {}
header.forEach(key => {
const type = predictType(key, data)
if (type === 'REAL' || type === 'INTEGER') {
// converting all data here to text
d = d.map(record => ({
...record,
[key]: type === 'REAL'
? parseFloat(record[key] as string)
: parseInt(record[key] as string)
}))
}
types[key] = type
})
return {
data: d,
types
}
}
export class SqlSealDatabase {
private savedDatabases: Record<string, any> = {}
db: Database.Database

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,22 @@
T__0=1
T__1=2
T__2=3
T__3=4
TABLE=5
FILE=6
WITH=7
AS=8
SELECT=9
ID=10
FILE_URL=11
WS=12
ANY=13
'='=1
'('=2
')'=3
','=4
'TABLE'=5
'file'=6
'WITH'=7
'AS'=8
'SELECT'=9

View file

@ -0,0 +1,56 @@
token literal names:
null
'='
'('
')'
','
'TABLE'
'file'
'WITH'
'AS'
'SELECT'
null
null
null
null
token symbolic names:
null
null
null
null
null
TABLE
FILE
WITH
AS
SELECT
ID
FILE_URL
WS
ANY
rule names:
T__0
T__1
T__2
T__3
TABLE
FILE
WITH
AS
SELECT
ID
FILE_URL
WS
ANY
channel names:
DEFAULT_TOKEN_CHANNEL
HIDDEN
mode names:
DEFAULT_MODE
atn:
[3, 51485, 51898, 1421, 44986, 20307, 1543, 60043, 49729, 2, 15, 82, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 8, 3, 8, 3, 8, 3, 8, 3, 8, 3, 9, 3, 9, 3, 9, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 7, 11, 66, 10, 11, 12, 11, 14, 11, 69, 11, 11, 3, 12, 6, 12, 72, 10, 12, 13, 12, 14, 12, 73, 3, 13, 6, 13, 77, 10, 13, 13, 13, 14, 13, 78, 3, 14, 3, 14, 2, 2, 2, 15, 3, 2, 3, 5, 2, 4, 7, 2, 5, 9, 2, 6, 11, 2, 7, 13, 2, 8, 15, 2, 9, 17, 2, 10, 19, 2, 11, 21, 2, 12, 23, 2, 13, 25, 2, 14, 27, 2, 15, 3, 2, 6, 5, 2, 67, 92, 97, 97, 99, 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 59, 67, 92, 94, 94, 97, 97, 99, 124, 5, 2, 11, 12, 15, 15, 34, 34, 2, 84, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 3, 29, 3, 2, 2, 2, 5, 31, 3, 2, 2, 2, 7, 33, 3, 2, 2, 2, 9, 35, 3, 2, 2, 2, 11, 37, 3, 2, 2, 2, 13, 43, 3, 2, 2, 2, 15, 48, 3, 2, 2, 2, 17, 53, 3, 2, 2, 2, 19, 56, 3, 2, 2, 2, 21, 63, 3, 2, 2, 2, 23, 71, 3, 2, 2, 2, 25, 76, 3, 2, 2, 2, 27, 80, 3, 2, 2, 2, 29, 30, 7, 63, 2, 2, 30, 4, 3, 2, 2, 2, 31, 32, 7, 42, 2, 2, 32, 6, 3, 2, 2, 2, 33, 34, 7, 43, 2, 2, 34, 8, 3, 2, 2, 2, 35, 36, 7, 46, 2, 2, 36, 10, 3, 2, 2, 2, 37, 38, 7, 86, 2, 2, 38, 39, 7, 67, 2, 2, 39, 40, 7, 68, 2, 2, 40, 41, 7, 78, 2, 2, 41, 42, 7, 71, 2, 2, 42, 12, 3, 2, 2, 2, 43, 44, 7, 104, 2, 2, 44, 45, 7, 107, 2, 2, 45, 46, 7, 110, 2, 2, 46, 47, 7, 103, 2, 2, 47, 14, 3, 2, 2, 2, 48, 49, 7, 89, 2, 2, 49, 50, 7, 75, 2, 2, 50, 51, 7, 86, 2, 2, 51, 52, 7, 74, 2, 2, 52, 16, 3, 2, 2, 2, 53, 54, 7, 67, 2, 2, 54, 55, 7, 85, 2, 2, 55, 18, 3, 2, 2, 2, 56, 57, 7, 85, 2, 2, 57, 58, 7, 71, 2, 2, 58, 59, 7, 78, 2, 2, 59, 60, 7, 71, 2, 2, 60, 61, 7, 69, 2, 2, 61, 62, 7, 86, 2, 2, 62, 20, 3, 2, 2, 2, 63, 67, 9, 2, 2, 2, 64, 66, 9, 3, 2, 2, 65, 64, 3, 2, 2, 2, 66, 69, 3, 2, 2, 2, 67, 65, 3, 2, 2, 2, 67, 68, 3, 2, 2, 2, 68, 22, 3, 2, 2, 2, 69, 67, 3, 2, 2, 2, 70, 72, 9, 4, 2, 2, 71, 70, 3, 2, 2, 2, 72, 73, 3, 2, 2, 2, 73, 71, 3, 2, 2, 2, 73, 74, 3, 2, 2, 2, 74, 24, 3, 2, 2, 2, 75, 77, 9, 5, 2, 2, 76, 75, 3, 2, 2, 2, 77, 78, 3, 2, 2, 2, 78, 76, 3, 2, 2, 2, 78, 79, 3, 2, 2, 2, 79, 26, 3, 2, 2, 2, 80, 81, 11, 2, 2, 2, 81, 28, 3, 2, 2, 2, 6, 2, 67, 73, 78, 2]

View file

@ -0,0 +1,22 @@
T__0=1
T__1=2
T__2=3
T__3=4
TABLE=5
FILE=6
WITH=7
AS=8
SELECT=9
ID=10
FILE_URL=11
WS=12
ANY=13
'='=1
'('=2
')'=3
','=4
'TABLE'=5
'file'=6
'WITH'=7
'AS'=8
'SELECT'=9

View file

@ -0,0 +1,131 @@
// Generated from SqlSealLang.g4 by ANTLR 4.9.0-SNAPSHOT
import { ATN } from "antlr4ts/atn/ATN";
import { ATNDeserializer } from "antlr4ts/atn/ATNDeserializer";
import { CharStream } from "antlr4ts/CharStream";
import { Lexer } from "antlr4ts/Lexer";
import { LexerATNSimulator } from "antlr4ts/atn/LexerATNSimulator";
import { NotNull } from "antlr4ts/Decorators";
import { Override } from "antlr4ts/Decorators";
import { RuleContext } from "antlr4ts/RuleContext";
import { Vocabulary } from "antlr4ts/Vocabulary";
import { VocabularyImpl } from "antlr4ts/VocabularyImpl";
import * as Utils from "antlr4ts/misc/Utils";
export class SqlSealLangLexer extends Lexer {
public static readonly T__0 = 1;
public static readonly T__1 = 2;
public static readonly T__2 = 3;
public static readonly T__3 = 4;
public static readonly TABLE = 5;
public static readonly FILE = 6;
public static readonly WITH = 7;
public static readonly AS = 8;
public static readonly SELECT = 9;
public static readonly ID = 10;
public static readonly FILE_URL = 11;
public static readonly WS = 12;
public static readonly ANY = 13;
// tslint:disable:no-trailing-whitespace
public static readonly channelNames: string[] = [
"DEFAULT_TOKEN_CHANNEL", "HIDDEN",
];
// tslint:disable:no-trailing-whitespace
public static readonly modeNames: string[] = [
"DEFAULT_MODE",
];
public static readonly ruleNames: string[] = [
"T__0", "T__1", "T__2", "T__3", "TABLE", "FILE", "WITH", "AS", "SELECT",
"ID", "FILE_URL", "WS", "ANY",
];
private static readonly _LITERAL_NAMES: Array<string | undefined> = [
undefined, "'='", "'('", "')'", "','", "'TABLE'", "'file'", "'WITH'",
"'AS'", "'SELECT'",
];
private static readonly _SYMBOLIC_NAMES: Array<string | undefined> = [
undefined, undefined, undefined, undefined, undefined, "TABLE", "FILE",
"WITH", "AS", "SELECT", "ID", "FILE_URL", "WS", "ANY",
];
public static readonly VOCABULARY: Vocabulary = new VocabularyImpl(SqlSealLangLexer._LITERAL_NAMES, SqlSealLangLexer._SYMBOLIC_NAMES, []);
// @Override
// @NotNull
public get vocabulary(): Vocabulary {
return SqlSealLangLexer.VOCABULARY;
}
// tslint:enable:no-trailing-whitespace
constructor(input: CharStream) {
super(input);
this._interp = new LexerATNSimulator(SqlSealLangLexer._ATN, this);
}
// @Override
public get grammarFileName(): string { return "SqlSealLang.g4"; }
// @Override
public get ruleNames(): string[] { return SqlSealLangLexer.ruleNames; }
// @Override
public get serializedATN(): string { return SqlSealLangLexer._serializedATN; }
// @Override
public get channelNames(): string[] { return SqlSealLangLexer.channelNames; }
// @Override
public get modeNames(): string[] { return SqlSealLangLexer.modeNames; }
public static readonly _serializedATN: string =
"\x03\uC91D\uCABA\u058D\uAFBA\u4F53\u0607\uEA8B\uC241\x02\x0FR\b\x01\x04" +
"\x02\t\x02\x04\x03\t\x03\x04\x04\t\x04\x04\x05\t\x05\x04\x06\t\x06\x04" +
"\x07\t\x07\x04\b\t\b\x04\t\t\t\x04\n\t\n\x04\v\t\v\x04\f\t\f\x04\r\t\r" +
"\x04\x0E\t\x0E\x03\x02\x03\x02\x03\x03\x03\x03\x03\x04\x03\x04\x03\x05" +
"\x03\x05\x03\x06\x03\x06\x03\x06\x03\x06\x03\x06\x03\x06\x03\x07\x03\x07" +
"\x03\x07\x03\x07\x03\x07\x03\b\x03\b\x03\b\x03\b\x03\b\x03\t\x03\t\x03" +
"\t\x03\n\x03\n\x03\n\x03\n\x03\n\x03\n\x03\n\x03\v\x03\v\x07\vB\n\v\f" +
"\v\x0E\vE\v\v\x03\f\x06\fH\n\f\r\f\x0E\fI\x03\r\x06\rM\n\r\r\r\x0E\rN" +
"\x03\x0E\x03\x0E\x02\x02\x02\x0F\x03\x02\x03\x05\x02\x04\x07\x02\x05\t" +
"\x02\x06\v\x02\x07\r\x02\b\x0F\x02\t\x11\x02\n\x13\x02\v\x15\x02\f\x17" +
"\x02\r\x19\x02\x0E\x1B\x02\x0F\x03\x02\x06\x05\x02C\\aac|\x06\x022;C\\" +
"aac|\x07\x02/;C\\^^aac|\x05\x02\v\f\x0F\x0F\"\"\x02T\x02\x03\x03\x02\x02" +
"\x02\x02\x05\x03\x02\x02\x02\x02\x07\x03\x02\x02\x02\x02\t\x03\x02\x02" +
"\x02\x02\v\x03\x02\x02\x02\x02\r\x03\x02\x02\x02\x02\x0F\x03\x02\x02\x02" +
"\x02\x11\x03\x02\x02\x02\x02\x13\x03\x02\x02\x02\x02\x15\x03\x02\x02\x02" +
"\x02\x17\x03\x02\x02\x02\x02\x19\x03\x02\x02\x02\x02\x1B\x03\x02\x02\x02" +
"\x03\x1D\x03\x02\x02\x02\x05\x1F\x03\x02\x02\x02\x07!\x03\x02\x02\x02" +
"\t#\x03\x02\x02\x02\v%\x03\x02\x02\x02\r+\x03\x02\x02\x02\x0F0\x03\x02" +
"\x02\x02\x115\x03\x02\x02\x02\x138\x03\x02\x02\x02\x15?\x03\x02\x02\x02" +
"\x17G\x03\x02\x02\x02\x19L\x03\x02\x02\x02\x1BP\x03\x02\x02\x02\x1D\x1E" +
"\x07?\x02\x02\x1E\x04\x03\x02\x02\x02\x1F \x07*\x02\x02 \x06\x03\x02\x02" +
"\x02!\"\x07+\x02\x02\"\b\x03\x02\x02\x02#$\x07.\x02\x02$\n\x03\x02\x02" +
"\x02%&\x07V\x02\x02&\'\x07C\x02\x02\'(\x07D\x02\x02()\x07N\x02\x02)*\x07" +
"G\x02\x02*\f\x03\x02\x02\x02+,\x07h\x02\x02,-\x07k\x02\x02-.\x07n\x02" +
"\x02./\x07g\x02\x02/\x0E\x03\x02\x02\x0201\x07Y\x02\x0212\x07K\x02\x02" +
"23\x07V\x02\x0234\x07J\x02\x024\x10\x03\x02\x02\x0256\x07C\x02\x0267\x07" +
"U\x02\x027\x12\x03\x02\x02\x0289\x07U\x02\x029:\x07G\x02\x02:;\x07N\x02" +
"\x02;<\x07G\x02\x02<=\x07E\x02\x02=>\x07V\x02\x02>\x14\x03\x02\x02\x02" +
"?C\t\x02\x02\x02@B\t\x03\x02\x02A@\x03\x02\x02\x02BE\x03\x02\x02\x02C" +
"A\x03\x02\x02\x02CD\x03\x02\x02\x02D\x16\x03\x02\x02\x02EC\x03\x02\x02" +
"\x02FH\t\x04\x02\x02GF\x03\x02\x02\x02HI\x03\x02\x02\x02IG\x03\x02\x02" +
"\x02IJ\x03\x02\x02\x02J\x18\x03\x02\x02\x02KM\t\x05\x02\x02LK\x03\x02" +
"\x02\x02MN\x03\x02\x02\x02NL\x03\x02\x02\x02NO\x03\x02\x02\x02O\x1A\x03" +
"\x02\x02\x02PQ\v\x02\x02\x02Q\x1C\x03\x02\x02\x02\x06\x02CIN\x02";
public static __ATN: ATN;
public static get _ATN(): ATN {
if (!SqlSealLangLexer.__ATN) {
SqlSealLangLexer.__ATN = new ATNDeserializer().deserialize(Utils.toCharArray(SqlSealLangLexer._serializedATN));
}
return SqlSealLangLexer.__ATN;
}
}

View file

@ -0,0 +1,109 @@
// Generated from SqlSealLang.g4 by ANTLR 4.9.0-SNAPSHOT
import { ParseTreeListener } from "antlr4ts/tree/ParseTreeListener";
import { ParseContext } from "./SqlSealLangParser";
import { StatementContext } from "./SqlSealLangParser";
import { TableStatementContext } from "./SqlSealLangParser";
import { QueryStatementContext } from "./SqlSealLangParser";
import { WithClauseContext } from "./SqlSealLangParser";
import { CteDefinitionContext } from "./SqlSealLangParser";
import { SelectStatementContext } from "./SqlSealLangParser";
import { SelectBodyContext } from "./SqlSealLangParser";
/**
* This interface defines a complete listener for a parse tree produced by
* `SqlSealLangParser`.
*/
export interface SqlSealLangListener extends ParseTreeListener {
/**
* Enter a parse tree produced by `SqlSealLangParser.parse`.
* @param ctx the parse tree
*/
enterParse?: (ctx: ParseContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.parse`.
* @param ctx the parse tree
*/
exitParse?: (ctx: ParseContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.statement`.
* @param ctx the parse tree
*/
enterStatement?: (ctx: StatementContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.statement`.
* @param ctx the parse tree
*/
exitStatement?: (ctx: StatementContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.tableStatement`.
* @param ctx the parse tree
*/
enterTableStatement?: (ctx: TableStatementContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.tableStatement`.
* @param ctx the parse tree
*/
exitTableStatement?: (ctx: TableStatementContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.queryStatement`.
* @param ctx the parse tree
*/
enterQueryStatement?: (ctx: QueryStatementContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.queryStatement`.
* @param ctx the parse tree
*/
exitQueryStatement?: (ctx: QueryStatementContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.withClause`.
* @param ctx the parse tree
*/
enterWithClause?: (ctx: WithClauseContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.withClause`.
* @param ctx the parse tree
*/
exitWithClause?: (ctx: WithClauseContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.cteDefinition`.
* @param ctx the parse tree
*/
enterCteDefinition?: (ctx: CteDefinitionContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.cteDefinition`.
* @param ctx the parse tree
*/
exitCteDefinition?: (ctx: CteDefinitionContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.selectStatement`.
* @param ctx the parse tree
*/
enterSelectStatement?: (ctx: SelectStatementContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.selectStatement`.
* @param ctx the parse tree
*/
exitSelectStatement?: (ctx: SelectStatementContext) => void;
/**
* Enter a parse tree produced by `SqlSealLangParser.selectBody`.
* @param ctx the parse tree
*/
enterSelectBody?: (ctx: SelectBodyContext) => void;
/**
* Exit a parse tree produced by `SqlSealLangParser.selectBody`.
* @param ctx the parse tree
*/
exitSelectBody?: (ctx: SelectBodyContext) => void;
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,80 @@
// Generated from SqlSealLang.g4 by ANTLR 4.9.0-SNAPSHOT
import { ParseTreeVisitor } from "antlr4ts/tree/ParseTreeVisitor";
import { ParseContext } from "./SqlSealLangParser";
import { StatementContext } from "./SqlSealLangParser";
import { TableStatementContext } from "./SqlSealLangParser";
import { QueryStatementContext } from "./SqlSealLangParser";
import { WithClauseContext } from "./SqlSealLangParser";
import { CteDefinitionContext } from "./SqlSealLangParser";
import { SelectStatementContext } from "./SqlSealLangParser";
import { SelectBodyContext } from "./SqlSealLangParser";
/**
* This interface defines a complete generic visitor for a parse tree produced
* by `SqlSealLangParser`.
*
* @param <Result> The return type of the visit operation. Use `void` for
* operations with no return type.
*/
export interface SqlSealLangVisitor<Result> extends ParseTreeVisitor<Result> {
/**
* Visit a parse tree produced by `SqlSealLangParser.parse`.
* @param ctx the parse tree
* @return the visitor result
*/
visitParse?: (ctx: ParseContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.statement`.
* @param ctx the parse tree
* @return the visitor result
*/
visitStatement?: (ctx: StatementContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.tableStatement`.
* @param ctx the parse tree
* @return the visitor result
*/
visitTableStatement?: (ctx: TableStatementContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.queryStatement`.
* @param ctx the parse tree
* @return the visitor result
*/
visitQueryStatement?: (ctx: QueryStatementContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.withClause`.
* @param ctx the parse tree
* @return the visitor result
*/
visitWithClause?: (ctx: WithClauseContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.cteDefinition`.
* @param ctx the parse tree
* @return the visitor result
*/
visitCteDefinition?: (ctx: CteDefinitionContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.selectStatement`.
* @param ctx the parse tree
* @return the visitor result
*/
visitSelectStatement?: (ctx: SelectStatementContext) => Result;
/**
* Visit a parse tree produced by `SqlSealLangParser.selectBody`.
* @param ctx the parse tree
* @return the visitor result
*/
visitSelectBody?: (ctx: SelectBodyContext) => Result;
}

View file

@ -0,0 +1,12 @@
import { describe, it, expect } from '@jest/globals'
import { parseLanguage } from './parser'
describe('Parser', () => {
it('should properly parse select only', () => {
expect(parseLanguage('SELECT * FROM files')).toEqual({
tables: [],
queryPart: 'SELECT * FROM files'
})
})
})

108
src/grammar/parser.ts Normal file
View file

@ -0,0 +1,108 @@
import { ANTLRErrorListener, CharStreams, CommonTokenStream, RecognitionException, Recognizer } from 'antlr4ts';
import { SqlSealLangLexer } from './SqlSealLangLexer';
import { QueryStatementContext, TableStatementContext, SqlSealLangParser, ParseContext, StatementContext, WithClauseContext, SelectStatementContext } from './SqlSealLangParser';
import { SqlSealLangVisitor } from './SqlSealLangVisitor';
import { AbstractParseTreeVisitor } from 'antlr4ts/tree/AbstractParseTreeVisitor';
export interface TableStatement {
name: string;
file: string;
url: string;
originalStatement: string;
}
export interface ParsedLanguage {
tables: TableStatement[];
queryPart: string | null;
}
class ErrorListener implements ANTLRErrorListener<any> {
private errors: string[] = [];
syntaxError(recognizer: Recognizer<any, any>, offendingSymbol: any, line: number, charPositionInLine: number, msg: string, e: RecognitionException | undefined): void {
this.errors.push(`Line ${line}:${charPositionInLine} ${msg}`);
}
hasErrors(): boolean {
return this.errors.length > 0;
}
getErrors(): string {
return this.errors.join('\n');
}
}
export class SqlSealLangVisitorImpl extends AbstractParseTreeVisitor<ParsedLanguage> implements SqlSealLangVisitor<ParsedLanguage> {
private tables: TableStatement[] = [];
private queryPart: string | null = null;
defaultResult(): ParsedLanguage {
return { tables: this.tables, queryPart: this.queryPart };
}
visitParse(ctx: ParseContext): ParsedLanguage {
ctx.statement().forEach(stmt => this.visit(stmt));
return this.defaultResult();
}
visitStatement(ctx: StatementContext): ParsedLanguage {
if (ctx.tableStatement()) {
this.visitTableStatement(ctx.tableStatement()!);
} else if (ctx.queryStatement()) {
this.visitQueryStatement(ctx.queryStatement()!);
}
return this.defaultResult();
}
visitTableStatement(ctx: TableStatementContext): ParsedLanguage {
const tableName = ctx.ID().text;
const fileUrl = ctx.FILE_URL().text;
// Reconstruct the original table statement with whitespace
const tableStmt = ctx.children!.map(child => child.text).join('');
this.tables.push({
name: tableName,
file: 'file',
url: fileUrl,
originalStatement: tableStmt
});
return this.defaultResult();
}
visitQueryStatement(ctx: QueryStatementContext): ParsedLanguage {
// Reconstruct the original query statement with whitespace and all characters
this.queryPart = ctx.children!.map(child => child.text).join('');
return this.defaultResult();
}
visitWithClause(ctx: WithClauseContext): ParsedLanguage {
// The WITH clause is already included in the queryPart, so we don't need to do anything here
return this.defaultResult();
}
visitSelectStatement(ctx: SelectStatementContext): ParsedLanguage {
// The SELECT statement is already included in the queryPart, so we don't need to do anything here
return this.defaultResult();
}
}
export function parseLanguage(input: string): ParsedLanguage {
const chars = CharStreams.fromString(input);
const lexer = new SqlSealLangLexer(chars);
const tokens = new CommonTokenStream(lexer);
const parser = new SqlSealLangParser(tokens);
const errorListener = new ErrorListener();
parser.removeErrorListeners();
parser.addErrorListener(errorListener);
const tree = parser.parse();
if (errorListener.hasErrors()) {
throw errorListener.getErrors()
}
const visitor = new SqlSealLangVisitorImpl();
return visitor.visit(tree);
}

64
src/models/sync.ts Normal file
View file

@ -0,0 +1,64 @@
import { SqlSealDatabase } from "src/database";
export interface SyncEntry {
filename: string;
url: string;
name: string;
syncedAt: string;
metadata: any;
}
export class SyncModel {
tableName = 'sqlseal_sync'
constructor(private db: SqlSealDatabase) {
this.db.connect().then(() =>
this.createTableIfNotExists()
)
}
createTableIfNotExists() {
this.db.createTable(this.tableName, {
'filename': 'TEXT',
'url': 'TEXT',
'name': 'TEXT',
'syncedAt': 'TEXT',
'metadata': 'JSON'
})
}
registerSync(filename: string, url: string, tableName: string) {
this.db.insertData(this.tableName, [{
filename,
url,
name: tableName,
syncedAt: Date.now(),
metadata: { }
}])
}
getSync(filename: string, tableName: string) {
const data = this.db.db.prepare(`
SELECT *
FROM sqlseal_sync
WHERE filename = @filename
AND name = @name
`).all({
filename: filename,
name: tableName,
})
if (data) {
return data[0] as SyncEntry
}
return null
}
removeSync(filename: string, tableName: string) {
this.db.db.prepare('DELETE FROM sqlseal_sync WHERE filename = :filename AND name = :name').run({
filename: filename,
name: tableName
})
}
}

View file

@ -1,6 +1,5 @@
import { BaseFrom, Parser, Select } from "node-sql-parser"
import { BaseFrom, From, Parser, Select, With } from "node-sql-parser"
import { generatePrefix } from "./hash"
import { table } from "console"
const isGlobal = (table: string, globalTables: string[]) => {
return globalTables.map(t =>
@ -9,7 +8,7 @@ const isGlobal = (table: string, globalTables: string[]) => {
}
const isBaseFrom = (from: From): from is BaseFrom => {
return !!from['table']
return Object.keys(from).includes('table')
}
export const prefixedIfNotGlobal = (tableName: string, globalTables: string[], prefix: string) => {
@ -20,7 +19,27 @@ export const prefixedIfNotGlobal = (tableName: string, globalTables: string[], p
return generatePrefix(prefix, tableName)
}
const updateSelect = (selectAst: Select, globalTables: string[], prefix: string) => {
const updateSelect = (selectAst: Select, globalTables: string[], prefix: string): { selectAst: Select, tables: string[] } => {
let cteGlobals: string[] = []
let withs = selectAst.with
let extraTables: string[] = []
if (selectAst.with) {
cteGlobals = selectAst.with.map(w => w.name.value)
const updatedWiths = selectAst.with.map(w => {
const { selectAst, tables } = updateSelect(w.stmt.ast, [...cteGlobals, ...globalTables], prefix)
extraTables.push(...tables)
return {
...w,
stmt: {
...w.stmt,
ast: selectAst
}
} satisfies With
})
withs = updatedWiths
}
if (!selectAst.from) {
return { selectAst: selectAst, tables: [] }
}
@ -29,11 +48,11 @@ const updateSelect = (selectAst: Select, globalTables: string[], prefix: string)
const updatedFrom = selectAst.from.map(from => {
if(isBaseFrom(from)) {
const t = prefixedIfNotGlobal(from.table, globalTables, prefix)
const t = prefixedIfNotGlobal(from.table, [...cteGlobals, ...globalTables], prefix)
tables.push(t)
return {
...from,
table: t
table: t,
}
}
return from
@ -42,9 +61,10 @@ const updateSelect = (selectAst: Select, globalTables: string[], prefix: string)
return {
selectAst: {
...selectAst,
from: updatedFrom
from: updatedFrom,
with: withs
},
tables
tables: [...tables, ...extraTables]
}
}

View file

@ -1,8 +1,8 @@
import { get } from "https"
import JSON5 from 'json5'
import { parse } from 'json5'
import * as fs from 'fs'
import { isNumeric } from "./database";
import { camelCase } from "lodash";
export const fetchBlobData = async (url: string, filePath: string) => {
return new Promise<void>((resolve, reject) => {
@ -90,10 +90,10 @@ export const predictJson = (data: Array<Record<string, unknown>>) => {
[k]: d[k]
}
}
const v = d[k].trim()
const v = (d[k] as string).trim()
if ((v.at(0) == '[' && v.at(-1) === ']') || (v.at(0) === '{' && v.at(-1) === '}')) {
try {
const d = JSON5.parse(v)
const d = parse(v)
return {
...o,
[k]: d
@ -106,4 +106,43 @@ export const predictJson = (data: Array<Record<string, unknown>>) => {
[k]: d[k]
}
}, {}))
}
export function isNumeric(str: string) {
if (typeof str != "string") return false // we only process strings!
return !isNaN(str as any) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
!isNaN(parseFloat(str)) // ...and ensure strings of whitespace fail
}
export function dataToCamelCase(data: Array<Record<string, unknown>>) {
return data.map(d => Object.keys(d).reduce((acc, k) => ({
...acc,
[camelCase(k)]: d[k]
}), {}))
}
export const toTypeStatements = (header: Array<string>, data: Array<Record<string, string>>) => {
let d: Array<Record<string, string | number>> = data
const types: Record<string, ReturnType<typeof predictType>> = {}
header.forEach(key => {
const type = predictType(key, data)
if (type === 'REAL' || type === 'INTEGER') {
// converting all data here to text
d = d.map(record => ({
...record,
[key]: type === 'REAL'
? parseFloat(record[key] as string)
: parseInt(record[key] as string)
}))
}
types[key] = type
})
return {
data: d,
types
}
}

View file

@ -1,5 +1,5 @@
import { describe, it, expect } from '@jest/globals'
import { predictType } from './utils'
import { predictJson, predictType } from './utils'
describe('Utils', () => {
@ -10,4 +10,10 @@ describe('Utils', () => {
it('should properly predict JSON type', () => {
expect(predictType('xxx', [{ 'xxx': { a: 'b' } }])).toEqual('JSON')
})
it('should properly predict json by parsing it', () => {
expect(predictJson([{ 'xxx': '["a", "b"]'}])).toEqual([{
xxx: ['a', 'b']
}])
})
})

View file

@ -1035,17 +1035,12 @@
resolved "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.5.tgz"
integrity sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==
"@types/node@*", "@types/node@^16.11.6":
version "16.18.96"
resolved "https://registry.npmjs.org/@types/node/-/node-16.18.96.tgz"
integrity sha512-84iSqGXoO+Ha16j8pRZ/L90vDMKX04QTYMTfYeE1WrjWaZXuchBehGUZEpNgx7JnmlrIHdnABmpjrQjhCnNldQ==
"@types/node@^18.0.0 || >=20.0.0":
version "20.12.7"
resolved "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz"
integrity sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==
"@types/node@*", "@types/node@^18.0.0 || >=20.0.0", "@types/node@^22.5.0":
version "22.5.2"
resolved "https://registry.npmjs.org/@types/node/-/node-22.5.2.tgz"
integrity sha512-acJsPTEqYqulZS/Yp/S3GgeE6GZ0qYODUR8aVr/DkhHQ8l9nd4j5x1/ZJy9/gHrRlFMqkO6i0I3E27Alu4jjPg==
dependencies:
undici-types "~5.26.4"
undici-types "~6.19.2"
"@types/node@^20.9.0":
version "20.14.15"
@ -1433,6 +1428,21 @@ ansi-styles@^5.0.0:
resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz"
integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
antlr4@^4.13.2:
version "4.13.2"
resolved "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz"
integrity sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg==
antlr4ts-cli@0.5.0-alpha.4:
version "0.5.0-alpha.4"
resolved "https://registry.npmjs.org/antlr4ts-cli/-/antlr4ts-cli-0.5.0-alpha.4.tgz"
integrity sha512-lVPVBTA2CVHRYILSKilL6Jd4hAumhSZZWA7UbQNQrmaSSj7dPmmYaN4bOmZG79cOy0lS00i4LY68JZZjZMWVrw==
antlr4ts@0.5.0-alpha.4:
version "0.5.0-alpha.4"
resolved "https://registry.npmjs.org/antlr4ts/-/antlr4ts-0.5.0-alpha.4.tgz"
integrity sha512-WPQDt1B74OfPv/IMS2ekXAKkTZIHl88uMetg6q3OTqgFxZ/dxDXI0EWLyZid/1Pe6hTftyg5N7gel5wNAGxXyQ==
anymatch@^3.0.3:
version "3.1.3"
resolved "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz"
@ -4693,6 +4703,11 @@ undici-types@~5.26.4:
resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz"
integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
undici-types@~6.19.2:
version "6.19.8"
resolved "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz"
integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==
unique-filename@^2.0.0:
version "2.0.1"
resolved "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz"