diff --git a/manifest.json b/manifest.json index 2f47b20..06d01cf 100644 --- a/manifest.json +++ b/manifest.json @@ -1,9 +1,9 @@ { "id": "hunchly", "name": "Hunchly", - "version": "1.0.1", + "version": "1.0.2", "minAppVersion": "0.15.0", - "description": "This plugin converts Hunchly's notes and captioned images into obsidian notes. Also adds selectors from hunchly as obsidian tags.", - "author": "sapperlabs", + "description": "This plugin converts an Hunchly case into Obsidian notes enriching the data along the way.", + "author": "Maltego GmBH", "isDesktopOnly": true } diff --git a/package-lock.json b/package-lock.json index 169d117..ac1a9fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,25 +1,34 @@ { - "name": "obsidian-sample-plugin", - "version": "1.0.0", + "name": "hunchly-obsidian-plugin", + "version": "1.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "obsidian-sample-plugin", - "version": "1.0.0", + "name": "hunchly-obsidian-plugin", + "version": "1.0.1", "license": "MIT", "dependencies": { + "compromise": "^14.14.4", + "he": "^1.2.0", + "html-to-text": "^9.0.5", + "htmlparser2": "^10.0.0", + "quoted-printable": "^1.0.1", "tmp": "^0.2.1", "unzipper": "^0.10.14" }, "devDependencies": { + "@types/he": "^1.2.3", + "@types/html-to-text": "^9.0.4", + "@types/htmlparser2": "^3.10.7", "@types/node": "^16.11.6", + "@types/quoted-printable": "^1.0.2", "@types/tmp": "^0.2.5", "@types/unzipper": "^0.10.8", "@typescript-eslint/eslint-plugin": "5.29.0", "@typescript-eslint/parser": "5.29.0", "builtin-modules": "3.3.0", - "esbuild": "0.17.3", + "esbuild": "^0.25.9", "obsidian": "latest", "tslib": "2.4.0", "typescript": "4.7.4" @@ -54,356 +63,446 @@ "w3c-keyname": "^2.2.4" } }, - "node_modules/@esbuild/android-arm": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.3.tgz", - "integrity": "sha512-1Mlz934GvbgdDmt26rTLmf03cAgLg5HyOgJN+ZGCeP3Q9ynYTNMn2/LQxIl7Uy+o4K6Rfi2OuLsr12JQQR8gNg==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.3.tgz", - "integrity": "sha512-XvJsYo3dO3Pi4kpalkyMvfQsjxPWHYjoX4MDiB/FUM4YMfWcXa5l4VCwFWVYI1+92yxqjuqrhNg0CZg3gSouyQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.3.tgz", - "integrity": "sha512-nuV2CmLS07Gqh5/GrZLuqkU9Bm6H6vcCspM+zjp9TdQlxJtIe+qqEXQChmfc7nWdyr/yz3h45Utk1tUn8Cz5+A==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.3.tgz", - "integrity": "sha512-01Hxaaat6m0Xp9AXGM8mjFtqqwDjzlMP0eQq9zll9U85ttVALGCGDuEvra5Feu/NbP5AEP1MaopPwzsTcUq1cw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.3.tgz", - "integrity": "sha512-Eo2gq0Q/er2muf8Z83X21UFoB7EU6/m3GNKvrhACJkjVThd0uA+8RfKpfNhuMCl1bKRfBzKOk6xaYKQZ4lZqvA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.3.tgz", - "integrity": "sha512-CN62ESxaquP61n1ZjQP/jZte8CE09M6kNn3baos2SeUfdVBkWN5n6vGp2iKyb/bm/x4JQzEvJgRHLGd5F5b81w==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.3.tgz", - "integrity": "sha512-feq+K8TxIznZE+zhdVurF3WNJ/Sa35dQNYbaqM/wsCbWdzXr5lyq+AaTUSER2cUR+SXPnd/EY75EPRjf4s1SLg==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.3.tgz", - "integrity": "sha512-CLP3EgyNuPcg2cshbwkqYy5bbAgK+VhyfMU7oIYyn+x4Y67xb5C5ylxsNUjRmr8BX+MW3YhVNm6Lq6FKtRTWHQ==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.3.tgz", - "integrity": "sha512-JHeZXD4auLYBnrKn6JYJ0o5nWJI9PhChA/Nt0G4MvLaMrvXuWnY93R3a7PiXeJQphpL1nYsaMcoV2QtuvRnF/g==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.3.tgz", - "integrity": "sha512-FyXlD2ZjZqTFh0sOQxFDiWG1uQUEOLbEh9gKN/7pFxck5Vw0qjWSDqbn6C10GAa1rXJpwsntHcmLqydY9ST9ZA==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.3.tgz", - "integrity": "sha512-OrDGMvDBI2g7s04J8dh8/I7eSO+/E7nMDT2Z5IruBfUO/RiigF1OF6xoH33Dn4W/OwAWSUf1s2nXamb28ZklTA==", - "cpu": [ - "loong64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.3.tgz", - "integrity": "sha512-DcnUpXnVCJvmv0TzuLwKBC2nsQHle8EIiAJiJ+PipEVC16wHXaPEKP0EqN8WnBe0TPvMITOUlP2aiL5YMld+CQ==", - "cpu": [ - "mips64el" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.3.tgz", - "integrity": "sha512-BDYf/l1WVhWE+FHAW3FzZPtVlk9QsrwsxGzABmN4g8bTjmhazsId3h127pliDRRu5674k1Y2RWejbpN46N9ZhQ==", + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", + "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", "cpu": [ "ppc64" ], "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", + "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", + "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", + "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", + "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", + "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", + "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", + "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", + "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", + "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", + "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", + "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", + "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", + "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.3.tgz", - "integrity": "sha512-WViAxWYMRIi+prTJTyV1wnqd2mS2cPqJlN85oscVhXdb/ZTFJdrpaqm/uDsZPGKHtbg5TuRX/ymKdOSk41YZow==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", + "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", "cpu": [ "riscv64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.3.tgz", - "integrity": "sha512-Iw8lkNHUC4oGP1O/KhumcVy77u2s6+KUjieUqzEU3XuWJqZ+AY7uVMrrCbAiwWTkpQHkr00BuXH5RpC6Sb/7Ug==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", + "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", "cpu": [ "s390x" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.3.tgz", - "integrity": "sha512-0AGkWQMzeoeAtXQRNB3s4J1/T2XbigM2/Mn2yU1tQSmQRmHIZdkGbVq2A3aDdNslPyhb9/lH0S5GMTZ4xsjBqg==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", + "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.3.tgz", - "integrity": "sha512-4+rR/WHOxIVh53UIQIICryjdoKdHsFZFD4zLSonJ9RRw7bhKzVyXbnRPsWSfwybYqw9sB7ots/SYyufL1mBpEg==", + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", + "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", "cpu": [ - "x64" + "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "netbsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.3.tgz", - "integrity": "sha512-cVpWnkx9IYg99EjGxa5Gc0XmqumtAwK3aoz7O4Dii2vko+qXbkHoujWA68cqXjhh6TsLaQelfDO4MVnyr+ODeA==", + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", + "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", + "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", "optional": true, "os": [ "openbsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.3.tgz", - "integrity": "sha512-RxmhKLbTCDAY2xOfrww6ieIZkZF+KBqG7S2Ako2SljKXRFi+0863PspK74QQ7JpmWwncChY25JTJSbVBYGQk2Q==", + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", + "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", + "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", + "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", "optional": true, "os": [ "sunos" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.3.tgz", - "integrity": "sha512-0r36VeEJ4efwmofxVJRXDjVRP2jTmv877zc+i+Pc7MNsIr38NfsjkQj23AfF7l0WbB+RQ7VUb+LDiqC/KY/M/A==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", + "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.3.tgz", - "integrity": "sha512-wgO6rc7uGStH22nur4aLFcq7Wh86bE9cOFmfTr/yxN3BXvDEdCSXyKkO+U5JIt53eTOgC47v9k/C1bITWL/Teg==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", + "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-x64": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.3.tgz", - "integrity": "sha512-FdVl64OIuiKjgXBjwZaJLKp0eaEckifbhn10dXWhysMJkWblg3OEEGKSIyhiD5RSgAya8WzP3DNkngtIg3Nt7g==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", + "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@eslint-community/eslint-utils": { @@ -456,6 +555,32 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/@eslint/js": { "version": "8.53.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.53.0.tgz", @@ -537,6 +662,19 @@ "node": ">= 8" } }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "selderee": "^0.11.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/@types/codemirror": { "version": "5.60.8", "resolved": "https://registry.npmjs.org/@types/codemirror/-/codemirror-5.60.8.tgz", @@ -546,12 +684,73 @@ "@types/tern": "*" } }, + "node_modules/@types/domhandler": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@types/domhandler/-/domhandler-2.4.5.tgz", + "integrity": "sha512-lANhC2grmFG1gBac/8sDAKdIXx+TzAdkJIAjEOSMA+qW3297ybACEbacJnG15aNYfrzDO6fdcoouokqAKsy6aQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/domutils": { + "version": "1.7.8", + "resolved": "https://registry.npmjs.org/@types/domutils/-/domutils-1.7.8.tgz", + "integrity": "sha512-iZGboDV79ibrO3D625p9yD+VgmMDnyJocdIRJvu9Xz66R8SHfOY/XNgdjY5SFoFiLgILceVfSLt7IUhlk1Vhhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/domhandler": "^2.4.0" + } + }, "node_modules/@types/estree": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.4.tgz", "integrity": "sha512-2JwWnHK9H+wUZNorf2Zr6ves96WHoWDJIftkcxPKsS7Djta6Zu519LarhRNljPXkpsZR2ZMwNCPeW7omW07BJw==", "dev": true }, + "node_modules/@types/he": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@types/he/-/he-1.2.3.tgz", + "integrity": "sha512-q67/qwlxblDzEDvzHhVkwc1gzVWxaNxeyHUBF4xElrvjL11O+Ytze+1fGpBHlr/H9myiBUaUXNnNPmBHxxfAcA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/html-to-text": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/@types/html-to-text/-/html-to-text-9.0.4.tgz", + "integrity": "sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/htmlparser2": { + "version": "3.10.7", + "resolved": "https://registry.npmjs.org/@types/htmlparser2/-/htmlparser2-3.10.7.tgz", + "integrity": "sha512-ycBs4PNr9rY9XFFp4WkP+M1UcO49ahn0+9b24cmIY6KWy0w35rW0G8+JTTe9Rp6Wnyqn5SEHZrhCBMa0TIOxBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/domhandler": "^2.4.3", + "@types/domutils": "*", + "@types/node": "*", + "domhandler": "^2.4.0" + } + }, + "node_modules/@types/htmlparser2/node_modules/domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/@types/htmlparser2/node_modules/domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "1" + } + }, "node_modules/@types/json-schema": { "version": "7.0.14", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.14.tgz", @@ -564,6 +763,13 @@ "integrity": "sha512-ZUGPWx5vKfN+G2/yN7pcSNLkIkXEvlwNaJEd4e0ppX7W2S8XAkdc/37hM4OUNJB9sa0p12AOvGvxL4JCPiz9DA==", "dev": true }, + "node_modules/@types/quoted-printable": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/quoted-printable/-/quoted-printable-1.0.2.tgz", + "integrity": "sha512-3B28oB1rRaZNb3N5dlxysm8lH1ujzvReDuYBiIO4jvpTIg9ksrILCNgPxSGVyTWE/qwuxzgHaVehwMK3CVqAtA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/tern": { "version": "0.23.6", "resolved": "https://registry.npmjs.org/@types/tern/-/tern-0.23.6.tgz", @@ -802,23 +1008,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "peer": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -892,21 +1081,23 @@ "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==" }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, + "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -998,6 +1189,20 @@ "dev": true, "peer": true }, + "node_modules/compromise": { + "version": "14.14.4", + "resolved": "https://registry.npmjs.org/compromise/-/compromise-14.14.4.tgz", + "integrity": "sha512-QdbJwronwxeqb7a5KFK/+Y5YieZ4PE1f7ai0vU58Pp4jih+soDCBMuKVbhDEPQ+6+vI3vSiG4UAAjTAXLJw1Qw==", + "license": "MIT", + "dependencies": { + "efrt": "2.7.0", + "grad-school": "0.0.5", + "suffix-thumb": "5.0.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1009,10 +1214,11 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "peer": true, "dependencies": { "path-key": "^3.1.0", @@ -1047,6 +1253,15 @@ "dev": true, "peer": true }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -1072,6 +1287,73 @@ "node": ">=6.0.0" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/dom-serializer/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/duplexer2": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", @@ -1080,41 +1362,67 @@ "readable-stream": "^2.0.2" } }, + "node_modules/efrt": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/efrt/-/efrt-2.7.0.tgz", + "integrity": "sha512-/RInbCy1d4P6Zdfa+TMVsf/ufZVotat5hCw3QXmWtjU+3pFEOvOQ7ibo3aIxyCJw2leIeAMjmPj+1SLJiCpdrQ==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/esbuild": { - "version": "0.17.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.3.tgz", - "integrity": "sha512-9n3AsBRe6sIyOc6kmoXg2ypCLgf3eZSraWFRpnkto+svt8cZNuKTkb1bhQcitBcvIqjNiK7K0J3KPmwGSfkA8g==", + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", + "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", "dev": true, "hasInstallScript": true, + "license": "MIT", "bin": { "esbuild": "bin/esbuild" }, "engines": { - "node": ">=12" + "node": ">=18" }, "optionalDependencies": { - "@esbuild/android-arm": "0.17.3", - "@esbuild/android-arm64": "0.17.3", - "@esbuild/android-x64": "0.17.3", - "@esbuild/darwin-arm64": "0.17.3", - "@esbuild/darwin-x64": "0.17.3", - "@esbuild/freebsd-arm64": "0.17.3", - "@esbuild/freebsd-x64": "0.17.3", - "@esbuild/linux-arm": "0.17.3", - "@esbuild/linux-arm64": "0.17.3", - "@esbuild/linux-ia32": "0.17.3", - "@esbuild/linux-loong64": "0.17.3", - "@esbuild/linux-mips64el": "0.17.3", - "@esbuild/linux-ppc64": "0.17.3", - "@esbuild/linux-riscv64": "0.17.3", - "@esbuild/linux-s390x": "0.17.3", - "@esbuild/linux-x64": "0.17.3", - "@esbuild/netbsd-x64": "0.17.3", - "@esbuild/openbsd-x64": "0.17.3", - "@esbuild/sunos-x64": "0.17.3", - "@esbuild/win32-arm64": "0.17.3", - "@esbuild/win32-ia32": "0.17.3", - "@esbuild/win32-x64": "0.17.3" + "@esbuild/aix-ppc64": "0.25.9", + "@esbuild/android-arm": "0.25.9", + "@esbuild/android-arm64": "0.25.9", + "@esbuild/android-x64": "0.25.9", + "@esbuild/darwin-arm64": "0.25.9", + "@esbuild/darwin-x64": "0.25.9", + "@esbuild/freebsd-arm64": "0.25.9", + "@esbuild/freebsd-x64": "0.25.9", + "@esbuild/linux-arm": "0.25.9", + "@esbuild/linux-arm64": "0.25.9", + "@esbuild/linux-ia32": "0.25.9", + "@esbuild/linux-loong64": "0.25.9", + "@esbuild/linux-mips64el": "0.25.9", + "@esbuild/linux-ppc64": "0.25.9", + "@esbuild/linux-riscv64": "0.25.9", + "@esbuild/linux-s390x": "0.25.9", + "@esbuild/linux-x64": "0.25.9", + "@esbuild/netbsd-arm64": "0.25.9", + "@esbuild/netbsd-x64": "0.25.9", + "@esbuild/openbsd-arm64": "0.25.9", + "@esbuild/openbsd-x64": "0.25.9", + "@esbuild/openharmony-arm64": "0.25.9", + "@esbuild/sunos-x64": "0.25.9", + "@esbuild/win32-arm64": "0.25.9", + "@esbuild/win32-ia32": "0.25.9", + "@esbuild/win32-x64": "0.25.9" } }, "node_modules/escape-string-regexp": { @@ -1238,6 +1546,24 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", @@ -1265,6 +1591,14 @@ "node": ">=4.0" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -1386,6 +1720,7 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true, + "license": "MIT", "peer": true }, "node_modules/fast-levenshtein": { @@ -1418,10 +1753,11 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, + "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -1577,6 +1913,15 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, + "node_modules/grad-school": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/grad-school/-/grad-school-0.0.5.tgz", + "integrity": "sha512-rXunEHF9M9EkMydTBux7+IryYXEZinRk6g8OBOGDBzo/qWJjhTxy86i5q7lQYpCLHN8Sqv1XX3OIOc7ka2gtvQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", @@ -1594,6 +1939,81 @@ "node": ">=8" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/html-to-text": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-9.0.5.tgz", + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.11.0", + "deepmerge": "^4.3.1", + "dom-serializer": "^2.0.0", + "htmlparser2": "^8.0.2", + "selderee": "^0.11.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/html-to-text/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/html-to-text/node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -1670,6 +2090,7 @@ "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.12.0" } @@ -1716,13 +2137,6 @@ "dev": true, "peer": true }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, - "peer": true - }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -1740,6 +2154,15 @@ "json-buffer": "3.0.1" } }, + "node_modules/leac": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.6.0.tgz", + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -1804,12 +2227,13 @@ } }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, + "license": "MIT", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -1953,6 +2377,19 @@ "node": ">=6" } }, + "node_modules/parseley": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "license": "MIT", + "dependencies": { + "leac": "^0.6.0", + "peberminta": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -1990,6 +2427,15 @@ "node": ">=8" } }, + "node_modules/peberminta": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==", + "license": "MIT", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -2022,6 +2468,7 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, + "license": "MIT", "peer": true, "engines": { "node": ">=6" @@ -2047,6 +2494,18 @@ } ] }, + "node_modules/quoted-printable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/quoted-printable/-/quoted-printable-1.0.1.tgz", + "integrity": "sha512-cihC68OcGiQOjGiXuo5Jk6XHANTHl1K4JLk/xlEJRTIXfy19Sg6XzB95XonYgr+1rB88bCpr7WZE7D7AlZow4g==", + "license": "MIT", + "dependencies": { + "utf8": "^2.1.0" + }, + "bin": { + "quoted-printable": "bin/quoted-printable" + } + }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -2097,6 +2556,8 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -2135,6 +2596,18 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, + "node_modules/selderee": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "license": "MIT", + "dependencies": { + "parseley": "^0.12.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -2228,6 +2701,12 @@ "dev": true, "peer": true }, + "node_modules/suffix-thumb": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/suffix-thumb/-/suffix-thumb-5.0.2.tgz", + "integrity": "sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==", + "license": "MIT" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2249,14 +2728,12 @@ "peer": true }, "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dependencies": { - "rimraf": "^3.0.0" - }, + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "license": "MIT", "engines": { - "node": ">=8.17.0" + "node": ">=14.14" } }, "node_modules/to-regex-range": { @@ -2264,6 +2741,7 @@ "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, + "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -2367,11 +2845,18 @@ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, + "license": "BSD-2-Clause", "peer": true, "dependencies": { "punycode": "^2.1.0" } }, + "node_modules/utf8": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/utf8/-/utf8-2.1.2.tgz", + "integrity": "sha512-QXo+O/QkLP/x1nyi54uQiG0XrODxdysuQvE5dtVqv7F5K2Qb6FsN+qbr6KhF5wQ20tfcV3VQp0/2x1e1MRSPWg==", + "license": "MIT" + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 88f1851..51ce5dc 100644 --- a/package.json +++ b/package.json @@ -12,18 +12,27 @@ "author": "", "license": "MIT", "devDependencies": { + "@types/he": "^1.2.3", + "@types/html-to-text": "^9.0.4", + "@types/htmlparser2": "^3.10.7", "@types/node": "^16.11.6", + "@types/quoted-printable": "^1.0.2", "@types/tmp": "^0.2.5", "@types/unzipper": "^0.10.8", "@typescript-eslint/eslint-plugin": "5.29.0", "@typescript-eslint/parser": "5.29.0", "builtin-modules": "3.3.0", - "esbuild": "0.17.3", + "esbuild": "^0.25.9", "obsidian": "latest", "tslib": "2.4.0", "typescript": "4.7.4" }, "dependencies": { + "compromise": "^14.14.4", + "he": "^1.2.0", + "html-to-text": "^9.0.5", + "htmlparser2": "^10.0.0", + "quoted-printable": "^1.0.1", "tmp": "^0.2.1", "unzipper": "^0.10.14" } diff --git a/src/fileModal.ts b/src/fileModal.ts index 7716aab..8056798 100644 --- a/src/fileModal.ts +++ b/src/fileModal.ts @@ -23,16 +23,16 @@ export class FileModal extends Modal { this.result.location = value })); - new Setting(contentEl) - .setName("Do you want to consolidate Hunchly notes by url?") - .addToggle((toggle) => - toggle.onChange((value) => { - if (value){ - this.result.consolidate = "true" - }else { - this.result.consolidate = "false" - } - })); + // new Setting(contentEl) + // .setName("Do you want to consolidate Hunchly notes by url?") + // .addToggle((toggle) => + // toggle.onChange((value) => { + // if (value){ + // this.result.consolidate = "true" + // }else { + // this.result.consolidate = "false" + // } + // })); new Setting(contentEl) .setName(this.inputString) .addButton((btn) => diff --git a/src/hunchly.ts b/src/hunchly.ts index 86aaf25..783421c 100644 --- a/src/hunchly.ts +++ b/src/hunchly.ts @@ -4,6 +4,12 @@ import { FileSystemAdapter, Plugin, TFile, Vault, Notice } from 'obsidian'; import * as path from 'path' import * as unzipper from 'unzipper'; import * as tmp from 'tmp'; +import { parseDocument } from "htmlparser2"; +import { textContent } from "domutils"; +import nlp from "compromise"; +import { decode } from "he"; +import * as qp from "quoted-printable"; +import { htmlToText } from "html-to-text"; interface IPage { title: string; @@ -26,7 +32,6 @@ interface IPhoto { photohash:string; photopath:string; } - export class Hunchly{ hunchlyExportPath: string; vaultPath: string; @@ -35,11 +40,20 @@ export class Hunchly{ hunchlyLocation: string consolidate: boolean - pages: Map - notes: Map + pages: Map + notes: Map selector: Map selectorHits: Map - taggedPhotos: Map + taggedPhotos: Map + extractedData: Map> + + regexMap: Record = { + // phone: /\+?\d[\d\-\s]{7,}\d/g, + ip: /\b(?:(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)(?:\.|$)){4}\b/g, // stricter IPv4 + email: /[A-Za-z0-9]+[A-Za-z0-9._%+-]*@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, // added \b for boundaries + // Facebook: either profile.php?id=123 OR /username (excluding common reserved paths) + social_media: /(?:https?:\/\/)?(?:www.)?(?:twitter|medium|facebook|vimeo|instagram)(?:.com\/)?([@a-zA-Z0-9-_]+)/img + }; constructor(hunchlyExportPath: string, vaultlocation: string, consolidate: string, plugin: Plugin) { this.hunchlyExportPath = hunchlyExportPath; @@ -70,14 +84,16 @@ export class Hunchly{ return } - const pages = await extractPages(extractionPath) - const notes = await extractNotes(extractionPath) - const photos = await extractPhotos(extractionPath) - const selectors = await extractSelectors(extractionPath) - const selectorHits = await extractSelectorsHits(extractionPath) + this.pages = await extractPages(extractionPath) + this.notes = await extractNotes(extractionPath) + this.taggedPhotos = await extractPhotos(extractionPath) + this.selector = await extractSelectors(extractionPath) + this.selectorHits = await extractSelectorsHits(extractionPath) + this.extractedData = await extractData(extractionPath) - await this.processNotes(notes, pages, selectors, selectorHits, extractionPath) - await this.processImages(photos, pages, selectors, selectorHits, extractionPath) + await this.processPages(extractionPath) + await this.processNotes(extractionPath) + await this.processImages(extractionPath) setTimeout(()=>{tempDir.removeCallback();}, 3000); @@ -97,12 +113,41 @@ export class Hunchly{ } } - private async processNotes(notes: Map, pages: Map, selectors: Map, selectorHits: Map, extractionPath : string){ + private async processPages(extractionPath : string){ + await this.createDirectoryIfNotExists(path.join("hunchly_captures", "screenshots")) + for (const [key, page] of this.pages) { + const selectorhits = this.selectorHits.get(key) + if (page) { + let title = page.title + let fileContent = "" + title = title.substring(0, 65).replace(/[&/\\#,+()$~%.'":*?<>{} ]/g,'_') + title = `${title}-captures-${key}` + + fileContent = "---\n" + fileContent = fileContent + `Date: ${page.date}\n` + fileContent = fileContent + `URL: ${page.url}\n` + fileContent = fileContent + "---\n" + + fileContent = await this.addDataExtractors(key, fileContent) + fileContent = await this.extractNamesAndLocationsFromMhtml(path.join(extractionPath, "pages", `${key}.mhtml`), fileContent) + if(selectorhits){ + fileContent = await addSelectors(selectorhits, this.selector, fileContent) + } + + fileContent = await this.addImages(path.join(extractionPath, "pages"), path.join(this.vaultPath, "hunchly_captures", "screenshots"), `${key}.jpeg`, fileContent) + fileContent = fileContent + "\n---\n" + + await this.createMDFile(`${title}.md`, fileContent, "hunchly_captures") + } + } + } + + private async processNotes(extractionPath : string){ await this.createDirectoryIfNotExists(path.join("hunchly_notes", "screenshots")) const urlMap = new Map() - for (const [key, value] of notes) { - const page = pages.get(value.pageid) - const selectorhits = selectorHits.get(value.pageid) + for (const [key, value] of this.notes) { + const page = this.pages.get(value.pageid) + const selectorhits = this.selectorHits.get(value.pageid) if (page) { let title = page.title let fileContent = "" @@ -115,7 +160,7 @@ export class Hunchly{ fileContent = fileContent + `URL: ${page.url}\n` fileContent = fileContent + "---\n" if(selectorhits){ - fileContent = await addSelectors(selectorhits, selectors, fileContent) + fileContent = await addSelectors(selectorhits, this.selector, fileContent) } } @@ -133,19 +178,19 @@ export class Hunchly{ await this.updateNoteFile(notePath, fileContent, "hunchly_notes") } } else { - await this.createNoteFile(`${title}.md`, fileContent, "hunchly_notes") + await this.createMDFile(`${title}.md`, fileContent, "hunchly_notes") urlMap.set(page.url, `${title}.md`) } } } } - private async processImages(photos: Map, pages: Map, selectors: Map, selectorHits: Map, extractionPath : string){ + private async processImages(extractionPath : string){ await this.createDirectoryIfNotExists(path.join("hunchly_captioned_images", "screenshots")) const urlMap = new Map() - for (const [key, value] of photos) { - const page = pages.get(value.pageid) - const selectorhits = selectorHits.get(value.pageid) + for (const [key, value] of this.taggedPhotos) { + const page = this.pages.get(value.pageid) + const selectorhits = this.selectorHits.get(value.pageid) if (page) { let title = page.title title = title.substring(0, 50).replace(/[&/\\#,+()$~%.'":*?<>{} ]/g,'_') @@ -159,7 +204,7 @@ export class Hunchly{ fileContent = fileContent + `HASH: ${value.photohash}\n` fileContent = fileContent + "---\n" if(selectorhits){ - fileContent = await addSelectors(selectorhits, selectors, fileContent) + fileContent = await addSelectors(selectorhits, this.selector, fileContent) } } @@ -167,7 +212,12 @@ export class Hunchly{ if(caption && caption != "null"){ fileContent = fileContent + `${caption}\n\n` } - fileContent = await this.addImages(path.join(extractionPath, "tagged_photos"), path.join(this.vaultPath, "hunchly_notes", "screenshots"), value.photopath, fileContent) + fileContent = await this.addImages( + path.join(extractionPath, "tagged_photos"), + path.join(this.vaultPath, "hunchly_captioned_images", "screenshots"), + value.photopath, + fileContent + ) fileContent = fileContent + "\n---\n" if (urlMap.has(page.url) && this.consolidate) { @@ -176,7 +226,7 @@ export class Hunchly{ await this.updateNoteFile(notePath, fileContent, "hunchly_captioned_images") } } else { - await this.createNoteFile(`${title}.md`, fileContent, "hunchly_captioned_images") + await this.createMDFile(`${title}.md`, fileContent, "hunchly_captioned_images") urlMap.set(page.url, `${title}.md`) } } @@ -211,10 +261,10 @@ export class Hunchly{ return fileContent } - private async createNoteFile(filename: string, content: string, location: string): Promise { + private async createMDFile(filename: string, content: string, location: string): Promise { try { - const notepath = path.join(this.hunchlyLocation, location, filename) - await this.vault.create(notepath, content) + const filepath = path.join(this.hunchlyLocation, location, filename) + await this.vault.create(filepath, content) } catch (error) { console.log(`Error creating the file in ${filename}: ${error}`); } @@ -233,9 +283,87 @@ export class Hunchly{ } } -} + private async addDataExtractors(pageId: number, fileContent: string): Promise{ + const matches = this.extractedData.get(pageId) + if (matches) { + for (const [key, data] of matches.entries()){ + fileContent = fileContent + `#### ${key}:\n\n` + for (const value of data){ + fileContent = fileContent + `[[${value}]]\t` + } + fileContent = fileContent + "\n\n" + } + } + + return fileContent + "\n\n---\n" + } + + private async extractNamesAndLocationsFromMhtml(filePath: string, fileContent: string): Promise{ + const raw = await fs_promise.readFile(filePath, "utf8"); + const htmlParts = raw.match(//gi); + + if (!htmlParts) { + return fileContent + }; + + let allText = ""; + + for (const m of htmlParts) { + // Decode quoted-printable first + const decodedQP = qp.decode(m.toString()); + + // Decode HTML entities (< & etc.) + const cleanHtml = decode(decodedQP.toString()); + + // Parse and extract text + const cleanText = htmlToText(cleanHtml, { + wordwrap: 130, + decodeEntities: true + }); + // allText += " " + textContent(dom); + allText += " " + cleanText; + } + + const dom = parseDocument(allText); + const text = textContent(dom) + + const doc = nlp(text); + const people = Array.from( + new Set(doc.people().out("array").map((p: string) => cleanString(p))) + ); + + const places = Array.from( + new Set(doc.places().out("array").map((p: string) => cleanString(p))) + ); + + const orgs = Array.from( + new Set(doc.organizations().out("array").map((p: string) => cleanString(p))) + ); + + fileContent = fileContent + `#### People:\n\n` + for (const value of people){ + fileContent = fileContent + `[[${value}]]\t` + } + fileContent = fileContent + "\n\n" + + fileContent = fileContent + `#### Location:\n\n` + for (const value of places){ + fileContent = fileContent + `[[${value}]]\t` + } + fileContent = fileContent + "\n\n" + + fileContent = fileContent + `#### Organizations:\n\n` + for (const value of orgs){ + fileContent = fileContent + `[[${value}]]\t` + } + fileContent = fileContent + "\n\n" + + return fileContent + "\n\n---\n" + } +} + async function addSelectors(selectorhits: number[], selectors: Map, fileContent: string): Promise{ fileContent = fileContent + "#### Selectors\n\n" selectorhits.forEach((selector)=>{ @@ -369,14 +497,73 @@ async function extractPages(zipFilePath: string): Promise> { return results } +async function extractData(zipFilePath: string) : Promise>> { + const dataMatches = path.join(zipFilePath, "case_data", "data_matches.json") + const dataExtractors = path.join(zipFilePath, "case_data", "data_extractors.json") + const result = new Map>() + + try { + interface Extractor { + ExtractorId: number; + PageId: number; + DataRecordId: number; + Data: string; + } + + const extractors = JSON.parse(await fs_promise.readFile(dataExtractors, 'utf8')); + const extractorMap = new Map() + if (Array.isArray(extractors)) { + extractors.forEach((item, index) => { + extractorMap.set(item.ID, item.Name) + }); + } + + if (extractorMap.size > 0) { + const matches: Extractor[]= JSON.parse(await fs_promise.readFile(dataMatches, 'utf8')); + if (Array.isArray(matches)) { + matches.forEach((item, index) => { + const extractorName = extractorMap.get(item.ExtractorId) + if (result && result.has(item.PageId)){ + const data= result.get(item.PageId) + if (extractorName && data && data.has(extractorName)){ + const val = data.get(extractorName) || [] + val.push(item.Data) + } else if (extractorName && data) { + data.set(extractorName, [item.Data]) + } else { + console.log("data is undefined") + } + } else { + const ex = new Map() + if (extractorName) { + result.set(item.PageId, ex.set(extractorName, [item.Data])) + } + } + }); + } + } + } catch (error) { + console.log(`Error parsing the selectors.json file: ${error}`); + } + + return result +} + async function copyImages(sourcePath: string, destinationPath: string): Promise { try { - const fileContent = await fs_promise.readFile(sourcePath); - await fs_promise.writeFile(destinationPath, fileContent); + await fs_promise.copyFile(sourcePath, destinationPath); } catch (error) { console.log(`Error copying the file: ${error}`); } } - - +function cleanString(str: string): string { + return str + .trim() + // Remove anything that’s not a letter, number, space, or basic punctuation + .replace(/[^a-zA-Z0-9\s.'-]/g, "") + // Collapse multiple spaces + .replace(/\s+/g, " ") + // Remove leading/trailing punctuation like commas, periods, dashes + .replace(/^[,.\-'\s]+|[,.\-'\s]+$/g, ""); +}