From 3bae9d6ff9fd349f1d4ceaf535c83171763a74b3 Mon Sep 17 00:00:00 2001 From: Mantano <91400447+mntno@users.noreply.github.com> Date: Mon, 10 Mar 2025 19:13:58 +0700 Subject: [PATCH] 1.0.0 --- .gitignore | 1 + LICENSE | 22 +- README.md | 49 ++- THIRD_PARTY_NOTICES | 49 +++ manifest.json | 2 +- package.json | 10 +- pnpm-lock.yaml | 77 ++-- src/CacheManager.ts | 884 ++++++++++++++++++++++++++++++------------ src/CacheMetadata.ts | 119 +++--- src/Environment.ts | 29 +- src/HtmlAssistant.ts | 364 ++++++++++------- src/InfoModal.ts | 100 +++++ src/ProcessingPass.ts | 159 ++++++++ src/Settings.ts | 89 ++++- src/Url.ts | 54 ++- src/main.ts | 627 +++++++++++++++--------------- tsconfig.json | 11 +- 17 files changed, 1791 insertions(+), 855 deletions(-) create mode 100644 THIRD_PARTY_NOTICES create mode 100644 src/InfoModal.ts create mode 100644 src/ProcessingPass.ts diff --git a/.gitignore b/.gitignore index be39a5d..bd4b472 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ node_modules .hotreload main.js +cache.json # Exclude sourcemaps *.map diff --git a/LICENSE b/LICENSE index 15ed5eb..d35e152 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,21 @@ -Copyright (C) 2025 by Mantano Bhikkhu. +MIT License -Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted. +Copyright (c) 2025 Mantano Bhikkhu -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 62908d1..41da0d4 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,53 @@ # Come Down -An Obsidian plugin that caches embedded external images to your local device. +An [Obsidian](https://obsidian.md) plugin that downloads external images embedded in your notes, allowing them to be reused when you reopen the note. Once downloaded (cached), images will load if you’re offline or if you open a synced copy of your vault on another device. The plugin doesn’t modify your notes, so if you disable or uninstall it, everything will work as it did before. + +## How to Use + +Embed external images as you normally would using a Markdown link. Here’s an example from [Obsidian Help](https://help.obsidian.md/embeds#Embed+an+image+in+a+note): + +```markdown +![250](https://publish-01.obsidian.md/access/f786db9fac45774fa4f0d8112e232d67/Attachments/Engelbart.jpg) +``` + +When the plugin detects an external image URL, it takes over the downloading process. Once the image is saved, the plugin will automatically load it from the cached file instead of the original URL. The next time you open the note, the cached version will be used. + +If your vault is synced through a file-syncing service like iCloud Drive or Dropbox (which treats your vault as a regular folder with files[^1]), cached images will sync across devices. + +However, if your vault is backed up to or synced via Git (e.g., GitHub, GitLab), the cache won’t be included. This is intentional for two reasons: First, Git is designed for versioning plain-text files, not storing binary data like images. Second, hosting services like GitHub and GitLab are meant for managing code and text-based content, not for storing personal images like a photo service. + +## Further Details + +### Cache Location + +Having the cache located inside the plugin’s folder serves several purposes: + +- It keeps the cache hidden from view while browsing the vault. +- It ensures the cache syncs along with the rest of the vault if stored in iCloud Drive or another file-syncing service that syncs the entire vault.[^1] +- It allows the cache to be automatically deleted if the plugin is uninstalled. +- A .gitignore file can be placed in the cache folder to exclude it from Git. Had it been in a visible folder, users might mistakenly think that files they add there would be committed. + +### Embedded Images Are Already Cached + +Obsidian is built on Electron, which embeds the Chromium web browser. Therefore, it has built-in caching just as any web browser. But this cache exists outside your vault. + +For example, if you copied your vault's root folder to a USB memory and opened it on another device, all embedded images would need to be downloaded again as each note is opened. Further, if that device were offline, they wouldn’t load at all. + +### Disabling + +If the plugin is disabled, everything will work as it did before the plugin was enabled — the embedded images will load by means of the underlying browser. Upon enabling it again, the plugin's already cached items will be used, bypassing the browser. + +Note that if an embedded image is removed from a note while the plugin is disabled, its potential cached file will not be removed until that specific note is opened again with the plugin enabled. If you want to disable the plugin you may delete the cache first from settings. + +[^1]: This excludes [Obsidian Sync](https://obsidian.md/sync). + + + -The cache is located within the plugin's folder, so the cached files: - -- will sync along with the rest of the vault if the vault is in iCloud Drive or any other file-syncing service that syncs the entire vault -- will be excluded from Git because a `.gitignore` file is placed in the cache folder -- will not sync with [Obsidian Sync](https://obsidian.md/sync) diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES new file mode 100644 index 0000000..9234523 --- /dev/null +++ b/THIRD_PARTY_NOTICES @@ -0,0 +1,49 @@ +THIRD-PARTY NOTICES + +This project’s functionality relies on the following third-party dependencies, each with its own license: + +License notice for image-size +----------------------------- + +https://github.com/image-size/image-size/blob/main/LICENSE + +The MIT License (MIT) + +Copyright © 2013-Present Aditya Yadav, http://netroy.in + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for xxhash-wasm +------------------------------ + +https://github.com/jungomi/xxhash-wasm/blob/main/LICENSE.md + +The MIT License (MIT) +===================== + +Copyright © 2017 Michael Jungo + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the “Software”), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/manifest.json b/manifest.json index 9133d20..da47d11 100644 --- a/manifest.json +++ b/manifest.json @@ -3,7 +3,7 @@ "name": "Come Down", "version": "1.0.0", "minAppVersion": "1.8.0", - "description": "Caches embedded external images to your local device without modifying your notes.", + "description": "Maintains a cache of your notes’ embedded external images.", "author": "mntno", "authorUrl": "https://github.com/mntno", "isDesktopOnly": false diff --git a/package.json b/package.json index a31f473..6562866 100644 --- a/package.json +++ b/package.json @@ -10,10 +10,8 @@ }, "keywords": [], "author": "mntno", - "license": "0BSD", - "devDependencies": { - "@codemirror/language": "^6.10.8", - "@codemirror/state": "^6.5.2", + "license": "MIT", + "devDependencies": { "@codemirror/view": "^6.36.4", "@eslint/js": "9.21.0", "@lezer/common": "^1.2.3", @@ -23,12 +21,12 @@ "builtin-modules": "5.0.0", "esbuild": "0.25.0", "globals": "16.0.0", + "image-size": "^2.0.1", "obsidian": "latest", - "ts-md5": "^1.3.1", "tslib": "2.8.1", "typescript": "5.8.2", "typescript-eslint": "8.26.0", - "image-size": "^2.0.0" + "xxhash-wasm": "1.1.0" }, "type": "module", "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2cc9a36..bff2967 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,6 @@ importers: .: devDependencies: - '@codemirror/language': - specifier: ^6.10.8 - version: 6.10.8 - '@codemirror/state': - specifier: ^6.5.2 - version: 6.5.2 '@codemirror/view': specifier: ^6.36.4 version: 6.36.4 @@ -42,14 +36,11 @@ importers: specifier: 16.0.0 version: 16.0.0 image-size: - specifier: ^2.0.0 - version: 2.0.0 + specifier: ^2.0.1 + version: 2.0.1 obsidian: specifier: latest version: 1.8.7(@codemirror/state@6.5.2)(@codemirror/view@6.36.4) - ts-md5: - specifier: ^1.3.1 - version: 1.3.1 tslib: specifier: 2.8.1 version: 2.8.1 @@ -59,12 +50,12 @@ importers: typescript-eslint: specifier: 8.26.0 version: 8.26.0(eslint@9.21.0)(typescript@5.8.2) + xxhash-wasm: + specifier: 1.1.0 + version: 1.1.0 packages: - '@codemirror/language@6.10.8': - resolution: {integrity: sha512-wcP8XPPhDH2vTqf181U8MbZnW+tDyPYy0UzVOa+oHORjyT+mhhom9vBd7dApJwoDz9Nb/a8kHjJIsuA/t8vNFw==} - '@codemirror/state@6.5.2': resolution: {integrity: sha512-FVqsPqtPWKVVL3dPSxy8wEF/ymIEuVzF1PK3VbUgrxXpJUSHQWWZz4JMToquRxnkw+36LTamCZG2iua2Ptq0fA==} @@ -221,8 +212,8 @@ packages: cpu: [x64] os: [win32] - '@eslint-community/eslint-utils@4.4.1': - resolution: {integrity: sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==} + '@eslint-community/eslint-utils@4.5.1': + resolution: {integrity: sha512-soEIOALTfTK6EjmKMMoLugwaP0rzkad90iIWd1hMO9ARkSAyjfMfkRRhLvD5qH7vvM0Cg72pieUfR6yh6XxC4w==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 @@ -278,12 +269,6 @@ packages: '@lezer/common@1.2.3': resolution: {integrity: sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==} - '@lezer/highlight@1.2.1': - resolution: {integrity: sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==} - - '@lezer/lr@1.4.2': - resolution: {integrity: sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==} - '@marijn/find-cluster-break@1.0.2': resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==} @@ -441,8 +426,8 @@ packages: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} - eslint-scope@8.2.0: - resolution: {integrity: sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==} + eslint-scope@8.3.0: + resolution: {integrity: sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} eslint-visitor-keys@3.4.3: @@ -545,8 +530,8 @@ packages: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} - image-size@2.0.0: - resolution: {integrity: sha512-HP07n1SpdIXGUL4VotUIOQz66MQOq8g7VN+Yj02YTVowqZScQ5i/JYU0+lkNr2pwt5j4hOpk94/UBV1ZCbS2fA==} + image-size@2.0.1: + resolution: {integrity: sha512-NI6NK/2zchlZopsQrcVIS7jxA0/rtIy74B+/rx5s7rKQyFebmQjZVhzxXgRZJROk+WhhOq+S6sUaODxp0L5hfg==} engines: {node: '>=16.x'} hasBin: true @@ -714,10 +699,6 @@ packages: peerDependencies: typescript: '>=4.8.4' - ts-md5@1.3.1: - resolution: {integrity: sha512-DiwiXfwvcTeZ5wCE0z+2A9EseZsztaiZtGrtSaY5JOD7ekPnR/GoIVD5gXZAlK9Na9Kvpo9Waz5rW64WKAWApg==} - engines: {node: '>=12'} - tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} @@ -755,21 +736,15 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + xxhash-wasm@1.1.0: + resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} snapshots: - '@codemirror/language@6.10.8': - dependencies: - '@codemirror/state': 6.5.2 - '@codemirror/view': 6.36.4 - '@lezer/common': 1.2.3 - '@lezer/highlight': 1.2.1 - '@lezer/lr': 1.4.2 - style-mod: 4.1.2 - '@codemirror/state@6.5.2': dependencies: '@marijn/find-cluster-break': 1.0.2 @@ -855,7 +830,7 @@ snapshots: '@esbuild/win32-x64@0.25.0': optional: true - '@eslint-community/eslint-utils@4.4.1(eslint@9.21.0)': + '@eslint-community/eslint-utils@4.5.1(eslint@9.21.0)': dependencies: eslint: 9.21.0 eslint-visitor-keys: 3.4.3 @@ -912,14 +887,6 @@ snapshots: '@lezer/common@1.2.3': {} - '@lezer/highlight@1.2.1': - dependencies: - '@lezer/common': 1.2.3 - - '@lezer/lr@1.4.2': - dependencies: - '@lezer/common': 1.2.3 - '@marijn/find-cluster-break@1.0.2': {} '@nodelib/fs.scandir@2.1.5': @@ -1013,7 +980,7 @@ snapshots: '@typescript-eslint/utils@8.26.0(eslint@9.21.0)(typescript@5.8.2)': dependencies: - '@eslint-community/eslint-utils': 4.4.1(eslint@9.21.0) + '@eslint-community/eslint-utils': 4.5.1(eslint@9.21.0) '@typescript-eslint/scope-manager': 8.26.0 '@typescript-eslint/types': 8.26.0 '@typescript-eslint/typescript-estree': 8.26.0(typescript@5.8.2) @@ -1120,7 +1087,7 @@ snapshots: escape-string-regexp@4.0.0: {} - eslint-scope@8.2.0: + eslint-scope@8.3.0: dependencies: esrecurse: 4.3.0 estraverse: 5.3.0 @@ -1131,7 +1098,7 @@ snapshots: eslint@9.21.0: dependencies: - '@eslint-community/eslint-utils': 4.4.1(eslint@9.21.0) + '@eslint-community/eslint-utils': 4.5.1(eslint@9.21.0) '@eslint-community/regexpp': 4.12.1 '@eslint/config-array': 0.19.2 '@eslint/core': 0.12.0 @@ -1148,7 +1115,7 @@ snapshots: cross-spawn: 7.0.6 debug: 4.4.0 escape-string-regexp: 4.0.0 - eslint-scope: 8.2.0 + eslint-scope: 8.3.0 eslint-visitor-keys: 4.2.0 espree: 10.3.0 esquery: 1.6.0 @@ -1242,7 +1209,7 @@ snapshots: ignore@5.3.2: {} - image-size@2.0.0: {} + image-size@2.0.1: {} import-fresh@3.3.1: dependencies: @@ -1379,8 +1346,6 @@ snapshots: dependencies: typescript: 5.8.2 - ts-md5@1.3.1: {} - tslib@2.8.1: {} type-check@0.4.0: @@ -1413,4 +1378,6 @@ snapshots: word-wrap@1.2.5: {} + xxhash-wasm@1.1.0: {} + yocto-queue@0.1.0: {} diff --git a/src/CacheManager.ts b/src/CacheManager.ts index 6532cb2..e4e8ab0 100644 --- a/src/CacheManager.ts +++ b/src/CacheManager.ts @@ -1,11 +1,10 @@ -import { App, normalizePath, requestUrl, Vault } from "obsidian"; -import { Md5 } from "ts-md5"; +import { normalizePath, requestUrl, Vault } from "obsidian"; import { imageSize } from 'image-size' -import { ISizeCalculationResult } from "image-size/dist/types/interface"; -import { CacheMetadata, CacheMetadataHash, CacheMetadataImage, CacheType, GetCacheKey } from "./CacheMetadata"; -import { ENV, Log } from "./Environment"; -import { Url } from "./Url"; -//import { XXHash3 } from 'xxhash-addon'; +import xxhash, { XXHashAPI } from "xxhash-wasm"; +import { CacheRoot, CacheMetadata, CacheMetadataImage, CacheRetainer, CacheType, EMPTY_CACHE_ROOT } from "CacheMetadata"; +import { ENV, Log } from "Environment"; +import { Url } from "Url"; + //#region @@ -14,41 +13,101 @@ export interface CacheRequest { * Unique key to identify the source of what is requested. * For an external image file, this would be the url. */ - key: string; + source: string; + + /** @todo Can do without. However, this makes each {@link CacheRequest} unique per requester. */ + requesterPath: string; } -/** - * Represents a cached file. - */ export interface CacheItem { - /** @deprecated Use CacheResult.request */ - request: CacheRequest; + + /** The absolute path to the item, which can vary per platform, e.g., `app://` on desktop but `capacitor://` on mobile. */ + resourcePath: string; + metadata: CacheMetadata; - /** Path to the cached file. */ - filePath: string; + + /** Whether this `item` was fetched from the cache. */ + fromCache: boolean; } /** * What you get after having requested a file from the {@link CacheManager}. */ -export interface CacheResult { - request: CacheRequest; - - item?: CacheItem; - - /** Whether this `item` was fetched from the cache. */ - fromCache: boolean; - - /** If {@link item} is set, this won't be. */ - error?: Error; +export class CacheResult { + /** + * @param request + * @param cacheKey + * @param item + * @param error If {@link item} is set, this won't be. + * @param fileExists If `undefined`, it's unknown whether the file exists. + */ + constructor( + public readonly request: CacheRequest, + public readonly cacheKey: string, + public readonly item: CacheItem | null, + public readonly error: Error | null, + public readonly fileExists?: boolean) { } } -function CacheResultFromError(error: Error, request: CacheRequest): CacheResult { - return { request, error, fromCache: false }; +export class CacheError extends Error { + constructor(public readonly cacheKey: string, message: string, options?: ErrorOptions) { + super(message, options); + this.name = "CacheError"; + } } -function CacheResultFromCache(item: CacheItem, request: CacheRequest, fromCache: boolean = true): CacheResult { - return { request, item, fromCache: fromCache }; +export class CacheTypeError extends CacheError { + constructor(error: TypeError) { + super("", error.message, { cause: error }); + this.name = "CacheTypeError"; + } +} + +/** + * In the event of a cache miss. + */ +export class CacheNotFoundError extends CacheError { + /** + * @param cacheKey + */ + constructor(cacheKey: string, cause?: Error) { + super(cacheKey, `Cache not found: ${cacheKey}`, { cause: cause }); + this.name = "CacheNotFoundError"; + } +} + +export class CacheFetchError extends CacheError { + public readonly isRetryable: boolean = false; + public readonly isInternetDisconnected: boolean = false; + + constructor(cacheKey: string, readonly error: Error | null, readonly info: { readonly sourceUrl: string, readonly statusCode?: number }) { + + super(cacheKey, `Failed to fetch cache: ${cacheKey}. Status: ${info?.statusCode ?? "Unknown"}`, { cause: error }); + + if (error) { + if (error.message && error.message.includes("net::ERR_NAME_NOT_RESOLVED")) { + Log(`Domain name resolution error: ${info.sourceUrl} - ${error.message}`); + } else if (error instanceof URIError) { + Log("Invalid URL provided."); + } else if (error.message && error.message.includes("net::ERR_INTERNET_DISCONNECTED")) { + Log("Network error: Could not connect to the server."); + this.isRetryable = true; + this.isInternetDisconnected = true; + } else { + Log("Request failed:", error); + } + } + else { + this.isRetryable = !this.info?.statusCode || (this.info?.statusCode >= 500 && this.info?.statusCode < 600); + } + } +} + +export interface CacheInfo { + summary: string; + numberOfFilesCached: number; + numberOfActualFilesWithoutAssociatedCacheKey: number; + numberOfCacheKeysWithoutAssociatedFile: number; } interface FileInfo { @@ -59,64 +118,384 @@ interface FileInfo { //#endregion export class CacheManager { - /** - * The in-memory cache. Populated gradually as cache requests are made. - */ - private cache: Record = {}; + + private metadataRoot: CacheRoot; + + private get cache(): Record { + return this.metadataRoot.items; + }; /** Ongoing downloads. As downloads complete, they will be removed. */ private readonly currentDownloads: Map = new Map(); private readonly vault: Vault; /** Root directory where cache is kept. Relative to vault root. */ private readonly cacheDir: string; + private readonly metadataFilePath: string; - constructor(vault: Vault, cacheDir: string) { + private static hasher: XXHashAPI; + + private readonly filePathsToOmitWhenClearingCache: string[] + + //#region + + private constructor(vault: Vault, cacheDir: string, metadataFilePath: string, filePathsToOmitWhenClearingCache: string[]) { this.vault = vault; this.cacheDir = cacheDir; + this.metadataFilePath = metadataFilePath; + this.filePathsToOmitWhenClearingCache = filePathsToOmitWhenClearingCache.includes(metadataFilePath) ? filePathsToOmitWhenClearingCache : [...filePathsToOmitWhenClearingCache, metadataFilePath]; } + public static async create(vault: Vault, cacheDir: string, metadataFilePath: string, filePathsToOmitWhenClearingCache: string[] = []) { + const instance = new this(vault, cacheDir, metadataFilePath, filePathsToOmitWhenClearingCache); + if (!this.hasher) + this.hasher = await xxhash(); // Seems to be quick. + + await instance.initCache(); // This can be called lazily if it turns out reading the json file takes too long time. + + return instance; + } + + /** + * Populates {@link cache} and {@link retainers} from JSON file. + */ + private async initCache() { + if (this.cacheInitiated) + return; + + if (this.initPromise) + return this.initPromise; // If there's an ongoing initialization, wait for it to complete. + + Log(`CacheManager:initCache`); + + this.initPromise = (async () => { + try { + await this.loadMetadata(); + this.cacheInitiated = true; + } catch (error) { + throw error; + } finally { + this.initPromise = null; // Reset when done. + } + })(); + + return this.initPromise; + } + private cacheInitiated = false; + private initPromise: Promise | null = null; + + //#endregion + /** * Call before making requests. * * @param request * @returns */ - validateRequest(request: CacheRequest): Error | undefined { - if (request.key.length == 0) + public validateRequest(request: CacheRequest): Error | undefined { + if (request.source.length == 0) return new Error(`The cache key is not set.`); - if (!Url.isExternal(request.key)) - return new Error(`The cache key must be an external Url (${request.key})`); + if (!Url.isExternal(request.source)) + return new Error(`The cache key must be an external Url (${request.source})`); + } - // if (!request.filePath || request.filePath.length == 0) - // return new Error(`Path to associated file not given.`); + //#region Retain/Release - // const associatedFile = this.app.vault.getFileByPath(request.filePath); - // if (!associatedFile) - // return new Error(`The file given does not exist: ${request.filePath}`); + /** + * - Only cached/downloaded resources can be retained + * - Only if they are actually refrenced should they be retained. + * - Each retainer only retains the cache item once, even if used more than once. + * + * Each time the retained caches are updated by a retainer a diff is made with the previous + * retained caches, which reveals references that were added and deleted. + * If a reference was deleted, the total retain count for that reference is checked, and if it's now 0, + * it is marked for deletion. + * + * @param requests + * @returns + */ + public async updateRetainedCaches(requests: CacheRequest[], retainerPath: string) { + + Log(`CacheManager:retainRequests ${requests.length}\n\tRetainer: ${retainerPath}`); + + let retainer: CacheRetainer = this.metadataRoot.retainers[retainerPath]; + + const oldCi = retainer ? [...retainer.ref] : []; + const newCi: string[] = []; + + for (const request of requests) { + const cacheKey = CacheManager.createCacheKeyFromRequest(request); + if (!newCi.includes(cacheKey)) { + newCi.push(cacheKey); + } + } + + const addedReferences = newCi.filter(key => !oldCi.includes(key)); + const removedReferences = oldCi.filter(key => !newCi.includes(key)); + + if (addedReferences.length == 0 && removedReferences.length == 0) { + Log(`\tNo change.`) + return; + } + + this.isMetadataDirty = true; + + // console.log(`Update: ` + newCi); + // console.log(`Retaining: ` + addedReferences); + // console.log(`Releasing: ` + removedReferences); + + if (retainer) { + retainer.ref = newCi; + } + else { + retainer = { ref: newCi } + this.metadataRoot.retainers[retainerPath] = retainer; + } + + const retainCount = this.retainCount(); + // console.log(this.retainCount()); + + for (const removedReference of removedReferences) { + if (removedReference in retainCount) { + const cacheKeyRetainCount = retainCount[removedReference]; + console.assert(cacheKeyRetainCount !== undefined, ""); + if (cacheKeyRetainCount === 0) { + await this.removeCacheItem(removedReference); + } + } + else + console.assert(""); + } + + // If it doesn't refernce anything anymore there's no need to keep it around. + if (retainer.ref.length == 0) + delete this.metadataRoot.retainers[retainerPath]; + } + + public renameRetainer(oldPath: string, path: string) { + Log(`CacheManager.renameRetainer\n\t${oldPath}`); + + let retainer: CacheRetainer = this.metadataRoot.retainers[oldPath]; + if (retainer) { + this.metadataRoot.retainers[path] = retainer; + delete this.metadataRoot.retainers[oldPath]; + this.isMetadataDirty = true; + } + } + + public async removeRetainer(path: string) { + Log(`CacheManager.removeRetainer\n\t${path}`); + + let retainer: CacheRetainer = this.metadataRoot.retainers[path]; + if (!retainer) { + // This can happen if a file which hasn't been open was deleted without opening it. + return; + } + + try { + const retainCount = this.retainCount(retainer) + for (const cacheKey of retainer.ref) { + // Release + retainCount[cacheKey]--; + + if (retainCount[cacheKey] == 0) { + await this.removeCacheItem(cacheKey); + } + } + + delete this.metadataRoot.retainers[path]; + this.isMetadataDirty = true; + } catch (error) { + console.error(error); + } + } + + private async removeCacheItem(cacheKey: string) { + Log(`CacheManager:removeCacheItem\n\tRemoving ${this.nameOfCachedFileFromMetadata(this.cache[cacheKey])}`); + await this.vault.adapter.remove(this.filePathToCachedFileFromMetadata(this.cache[cacheKey])); + delete this.cache[cacheKey]; } /** - * Returns immediately if the {@link CacheItem|cache} already exists in memory. If not, then — if all its referenced files also exist — it will be added to memory and returned. + * First registers all in-memory cache keys (if the {@link byRetainer} parameter is passed, only registers keys referenced by that retainer), + * then goes through all in-memory retainers and counts the number of total references + * for each registered cache key. * - * - Call {@link validateRequest} first. - * - This method is `async` because it does file I/O. - * - {@link getCache} calls this method before downloading. If it's already known that the cache doesn't exist, let {@link getCache} know. - * - * @param request - * @param onMetadata Will be called if no {@link CacheItem} was retrieved, but its metadata was. Usually you'd get it from {@link CacheItem}. - * @returns `null` if the requested {@link CacheItem} could not be retrieved. + * @param byRetainer + * @returns A `Record` where the key is the cache key and the value its retain count. */ - public async existingCachedItem(request: CacheRequest, onMetadata?: (metadata: CacheMetadata) => Promise): Promise { + public retainCount(byRetainer?: CacheRetainer) { + // Get retain counts on all caches used. + const retainCounts: Record = {}; - const cacheID = CacheManager.createCacheKey(request); + if (byRetainer) { + for (const cacheKey of byRetainer.ref) { + retainCounts[cacheKey] = 0; + } + } + else { + for (const cacheKey of Object.keys(this.cache)) { + retainCounts[cacheKey] = 0; + } + } - // If the cache exists in memory, it's assumed that the files exist as well (see below), so can return immediately. - const inMemoryCache = this.cache[cacheID]; - if (inMemoryCache) - return inMemoryCache; + for (const retainer of Object.values(this.metadataRoot.retainers)) { + for (const cacheKey of retainer.ref) + if (cacheKey in retainCounts) + retainCounts[cacheKey]++; + } - // Not in memory, get from drive. + return retainCounts; + } + + public async info(callback: (info: CacheInfo) => void) { + + const cacheKeys = Object.keys(this.cache); + + // Actual files on disk + const actualCachedFilePaths = await this.actualCachedFilePaths(); + + // There is a cache key but no file with the same name. + const cacheKeysWithoutAssociatedFile = []; + for (const cacheKey of cacheKeys) { + const cacheKeyMetadata: CacheMetadata = this.cache[cacheKey]; + let found = false; + for (const associatedFilePath of actualCachedFilePaths) { + if (associatedFilePath === this.filePathToCachedFileFromMetadata(cacheKeyMetadata, cacheKey)) { + found = true; + break; + } + } + if (!found) + cacheKeysWithoutAssociatedFile.push(cacheKey); + } + + // There is an actual file on disk but no corresponding cache key. + const actualFileWithoutAssociatedCacheKey = []; + for (const actualFilePath of actualCachedFilePaths) { + let found = false; + for (const cacheKey of cacheKeys) { + const cacheKeyMetadata: CacheMetadata = this.cache[cacheKey]; + if (actualFilePath === this.filePathToCachedFileFromMetadata(cacheKeyMetadata, cacheKey)) { + found = true; + break; + } + } + if (!found) + actualFileWithoutAssociatedCacheKey.push(actualFilePath.split("/").pop()); + } + + // + const retainers: CacheRetainer[] = Object.values(this.metadataRoot.retainers); + const numberOfRetainers = retainers.length; + + // Here the Mardown file has been deleted but its still exists as a retainer. + const retainersWithoutActualFile = []; + for (const actualFilePath in this.metadataRoot.retainers) { + if (!await this.vault.adapter.exists(actualFilePath)) + retainersWithoutActualFile.push(actualFilePath); + } + + const retainersWithoutReferences = []; + for (const retainer of retainers) { + if (!retainer.ref || retainer.ref.length == 0) + retainersWithoutReferences.push(retainer); + } + + // There are cache keys that aren't referenced by any retainer. + // TODO: Files not marked as retainer + const cacheKeysWithoutAnyRetainer: string[] = []; + const retainCount = this.retainCount(); + + for (const cacheKey of cacheKeys) { + if (cacheKey in retainCount) { + if (retainCount[cacheKey] == 0) // If the retain count is zero there is no retainer referencing it. + cacheKeysWithoutAnyRetainer.push(cacheKey); + } + else // If there's a cache key that's not found in the retain count record, it means that there's no retainer that references the cache key. + cacheKeysWithoutAnyRetainer.push(cacheKey); + } + + const numberOfCacheKeysWithoutAssociatedFile = cacheKeysWithoutAssociatedFile.length; + const numberOfActualFileWithoutAssociatedCacheKey = actualFileWithoutAssociatedCacheKey.length; + + let summary = ""; + summary += `Cache items without file: ${numberOfCacheKeysWithoutAssociatedFile}${numberOfCacheKeysWithoutAssociatedFile > 0 ? " 🛑" : ""}${numberOfCacheKeysWithoutAssociatedFile > 0 ? `: ${cacheKeysWithoutAssociatedFile.join(", ")}` : ""}\n`; + summary += `Files whithout cache items: ${numberOfActualFileWithoutAssociatedCacheKey}${numberOfActualFileWithoutAssociatedCacheKey > 0 ? " 🛑" : ""}${actualFileWithoutAssociatedCacheKey.length > 0 ? `: ${actualFileWithoutAssociatedCacheKey.join(", ")}` : ""}\n`; + summary += `Cache items without retainer: ${cacheKeysWithoutAnyRetainer.length}${cacheKeysWithoutAnyRetainer.length > 0 ? " 🛑" : ""}${cacheKeysWithoutAnyRetainer.length > 0 ? `: ${cacheKeysWithoutAnyRetainer.join(", ")}` : ""}\n\n`; + summary += `Retainers without references: ${retainersWithoutReferences.length}${retainersWithoutReferences.length > 0 ? " 🛑" : ""}\n`; + summary += `Retainers without file: ${retainersWithoutActualFile.length}${retainersWithoutActualFile.length > 0 ? " 🛑" : ""}\n`; + + callback({ + summary: summary, + numberOfFilesCached: actualCachedFilePaths.length, + numberOfCacheKeysWithoutAssociatedFile: numberOfCacheKeysWithoutAssociatedFile, + numberOfActualFilesWithoutAssociatedCacheKey: numberOfActualFileWithoutAssociatedCacheKey, + }); + } + + //#endregion + + /** + * Get {@link CacheMetadata} if in cache. + * Will do nothing if the request doesn't exist in the cache. + * @param request + * @param [ignoreMissingFile=false] If set, will return the metadata without checking if the associated cache file actually exists. + * @returns Will also return `null` if the actual file does not exist. + */ + public async existingCache(request: CacheRequest, ignoreMissingFile: boolean = false, cacheKey?: string): Promise { + if (!this.cacheInitiated) + await this.initCache() + + cacheKey = cacheKey ?? CacheManager.createCacheKeyFromRequest(request); + const metadata = this.cache[cacheKey]; + if (metadata) { + if (ignoreMissingFile || await this.vault.adapter.exists(this.filePathToCachedFileFromMetadata(metadata), true)) + return new CacheResult(request, cacheKey, this.createCacheItem(metadata, true), null, ignoreMissingFile ? undefined : true); + else + return new CacheResult(request, cacheKey, null, new CacheNotFoundError(cacheKey), false); + } + + return new CacheResult(request, cacheKey, null, new CacheNotFoundError(cacheKey), undefined); + } + + private createCacheItem(metadata: CacheMetadata, fromCache: boolean): CacheItem { + return { + resourcePath: this.vault.adapter.getResourcePath(this.filePathToCachedFileFromMetadata(metadata)), + metadata: metadata, + fromCache: fromCache + }; + } + + /** + * + * @param request + * @param force Download immediately without checking if cache already exists. + * @param callback + * @returns + */ + public async getCache(request: CacheRequest, force: boolean, callback: (result: CacheResult) => void): Promise { + if (!this.cacheInitiated) + await this.initCache(); + + const validationError = this.validateRequest(request); + if (validationError) { + callback(new CacheResult(request, CacheManager.createCacheKeyFromRequest(request), null, validationError)); + return; + } + + const download = async () => { + const sourceFileInfo = Url.extractFilenameAndExtension(request.source) as FileInfo | null; + if (!sourceFileInfo) + callback(new CacheResult(request, CacheManager.createCacheKeyFromRequest(request), null, new Error(`Failed to extract fileInfo from source url.`))); + else + callback(await this.fetchNewCache(request, sourceFileInfo)); + }; + + if (force) { + await download(); + return; + } // TODO: If assuming that, if the requested item is downloading, it doesn't exist on disk, then we could return null here. But can also proceed anyway. // if (this.cacheRequests.get(cacheID)) { @@ -124,91 +503,18 @@ export class CacheManager { // return null; // } - // Even if a metadata file for the cache exists, also make sure the actual file exists. - // Once the in-memory cache has been updated subsequent calls to this method will not reach this part. - // In cases when both files don't exist, however, subsequent calls to this method will continue to check for these files until the in-memory cache has been updated from having been downloaded. - const metadataFileExists = await this.vault.adapter.exists(this.filePathToAssociatedMetadata(request), true); - if (metadataFileExists) { - - const metadataFileContent: string = await this.vault.adapter.read(this.filePathToAssociatedMetadata(request)); - const metadata: CacheMetadata = JSON.parse(metadataFileContent); - - // Only set in-memory cache if the actual file exists. - const cachedFileExists = await this.vault.adapter.exists(this.filePathToCachedFileFromMetadata(metadata), true); - if (cachedFileExists) { - // TODO: Multiple files waiting requesting the same cache. - // this.cache[cacheID] = cacheResult.item might overwrite `CacheRequest.filePath`. - - console.assert(GetCacheKey(metadata.hash) === cacheID, `metadata.hash.keyMD5 !== cacheID`); - Log(`CacheManager:existingCachedItem\n\tGot cached result from metadata file: ${metadata.hash.keyMD5}`); - return this.setCache(request, metadata); - } - else { - await onMetadata?.(metadata); - } + const result = await this.existingCache(request); + if (result.item) { + Log(`CacheManager:getCache\n\tGot cache for cacheKey: ${result.cacheKey}`); + callback(result); } else { - Log(`CacheManager:existingCachedItem\n\tMetadata file for ${Url.extractFilenameAndExtension(request.key)?.filename} does not exist.`); - } - - return null; - } - - /** - * - * @param request - * @param callback - * @param force Omit checking with {@link existingCachedItem} whether the cache already exists; download immediately. - * @returns - */ - public async getCache(request: CacheRequest, callback: (result: CacheResult) => void, force: boolean = false): Promise { - - const validationError = this.validateRequest(request); - if (validationError) { - callback(CacheResultFromError(validationError, request)); - return; - } - - const item = force ? null : await this.existingCachedItem(request, async (metadata) => { - Log(`CacheManager:getCache\n\tMetadata exists but the actual file ${this.nameOfCachedFileFromMetadata(metadata)} doesn't. Download again.`) - const sourceFileInfo: FileInfo = { filename: metadata.file.name, extension: metadata.file.ext }; - callback(await this.fetchNewCache(request, sourceFileInfo)); - }); - - if (item) { - callback(CacheResultFromCache(item, request)); - } - else { - const sourceFileInfo = Url.extractFilenameAndExtension(request.key) as FileInfo | null; - if (!sourceFileInfo) - callback(CacheResultFromError(new Error(`Failed to extract fileInfo from source url.`), request)); - else - callback(await this.fetchNewCache(request, sourceFileInfo)); + await download(); } } /** - * Creates a {@link CacheItem}, adds it to the in-memory cache, and then returns it. - * @param request - * @param metadata - * @returns - */ - private setCache(request: CacheRequest, metadata: CacheMetadata): CacheItem { - const cacheItem: CacheItem = { - request: request, - metadata: metadata, - filePath: this.vault.adapter.getResourcePath(this.filePathToCachedFile(request, metadata.file.ext)), - }; - this.cache[GetCacheKey(metadata.hash)] = cacheItem; - - return cacheItem; - } - - /** - * When a {@link request} does not yield a local result, call this, which will: - * - * - call {@link download} to download and write files to storage. - * - save the result to the in-memory {@link cache} + * When a {@link request} does not yield a local result, call this method to download. * * @param request * @param sourceFileInfo @@ -216,29 +522,22 @@ export class CacheManager { * @returns */ private async fetchNewCache(request: CacheRequest, sourceFileInfo: FileInfo): Promise { - let cacheID: string | undefined; + let cacheKey: string | undefined; let result: CacheResult | undefined; try { - cacheID = CacheManager.createCacheKey(request); + cacheKey = CacheManager.createCacheKeyFromRequest(request); // TODO: Multiple Files Waiting Requesting the Same Cache - this.currentDownloads.set(cacheID, request); + this.currentDownloads.set(cacheKey, request); - const metadata: CacheMetadata | null = await this.download(request, sourceFileInfo); - if (metadata) { - Log(`CacheManager:fetchNewCache\n\tDownloaded and cached ${this.nameOfCachedFileFromMetadata(metadata)}`); - result = CacheResultFromCache(this.setCache(request, metadata), request, false); - } - else { - result = CacheResultFromError(new Error(`Failed to download file.`), request); - } + result = await this.download(request, sourceFileInfo); } catch (error) { - result = CacheResultFromError(error, request); + result = new CacheResult(request, cacheKey ?? "", null, error); } finally { - if (cacheID) - this.currentDownloads.delete(cacheID); + if (cacheKey) + this.currentDownloads.delete(cacheKey); } return result; @@ -249,21 +548,77 @@ export class CacheManager { * * @param filePathsToOmit Array of normalized paths to files to not delete from disk. */ - async clearCached(filePathsToOmit: string[], callback?: (error?: Error) => void) { + async clearCached(callback?: (error?: Error) => void) { try { - this.cache = {}; // Clear in-memory first in case something goes wrong below. await this.cancelAllOngoing(); - const listed = await this.vault.adapter.list(this.cacheDir); - for (const filePath of listed.files) { - if (!filePathsToOmit.includes(filePath)) + + for (const filePath of await this.actualCachedFilePaths()) { + if (!this.filePathsToOmitWhenClearingCache.includes(filePath)) await this.vault.adapter.remove(filePath); } + + await this.resetMetadata(); + callback?.(); } catch (error) { callback?.(error); } } + public async actualCachedFilePaths() { + const listed = await this.vault.adapter.list(this.cacheDir); + return listed + .files + .filter((filePath) => !this.filePathsToOmitWhenClearingCache.includes(filePath)) + } + + public debug() { + return { + loadMetadata: async () => await this.loadMetadata(), + saveMetadata: async () => await this.saveMetadata(), + }; + } + + public async loadMetadata() { + const metadataFileExists = await this.vault.adapter.exists(this.metadataFilePath, true); + if (metadataFileExists) { + const metadataFileContent: string = await this.vault.adapter.read(this.metadataFilePath); + try { + this.metadataRoot = Object.assign({}, EMPTY_CACHE_ROOT, JSON.parse(metadataFileContent)); + } catch (error) { + console.error("Failed to read metadata. Clearing cache.", error); + await this.clearCached(); + } + } + else { + this.resetMetadata(); + } + } + + /** + * Empties {@link metadataRoot} and its associated file at {@link metadataFilePath}. + * If they don't exist, they will be created. + */ + private async resetMetadata() { + this.metadataRoot = Object.assign({}, EMPTY_CACHE_ROOT); + await this.saveMetadata(); + } + + /** + * @throws {Error} If writing to the storage fails. + */ + private async saveMetadata() { + Log(`CacheManager: saveMetadata`) + await this.vault.adapter.write(this.metadataFilePath, ENV.dev ? JSON.stringify(this.metadataRoot, null, 2) : JSON.stringify(this.metadataRoot)); + this.isMetadataDirty = false; + } + + public saveMetadataIfDirty(): Promise | null { + Log(`CacheManager: saveMetadataIfDirty ${this.isMetadataDirty}`); + return this.isMetadataDirty ? this.saveMetadata() : null; + } + private isMetadataDirty: boolean = false; + /** * Aborts current download requests for the specified file. * @todo @@ -280,8 +635,16 @@ export class CacheManager { await sleep(100); } - public static cacheKeyFromOriginalSrc(src: string) { - return Md5.hashStr(src.trim()); + public static createCacheKeyFromOriginalSrc(src: string) { + return this.hashString(src); + } + + private static hashString(text: string) { + return CacheManager.hasher.h64ToString(text.trim()); + } + + private static hashBinary(bytes: Uint8Array) { + return CacheManager.hasher.h64Raw(bytes).toString(16).padStart(16, '0'); } /** @@ -289,136 +652,167 @@ export class CacheManager { * @param request * @returns */ - private static createCacheKey(request: CacheRequest) { - return Md5.hashStr(request.key); + private static createCacheKeyFromRequest(request: CacheRequest) { + return CacheManager.createCacheKeyFromOriginalSrc(request.source) } - private filePathToAssociatedMetadata(request: CacheRequest) { - return normalizePath(`${this.cacheDir}/${CacheManager.createCacheKey(request)}.json`); + public static createCacheKeyFromMetadata(metadata: CacheMetadata) { + return CacheManager.createCacheKeyFromOriginalSrc(metadata.f.s); } + // private filePathToAssociatedMetadata(request: CacheRequest) { + // return normalizePath(`${this.cacheDir}/${CacheManager.createCacheKey(request)}.json`); + // } + private filePathToCachedFile(request: CacheRequest, extension: string) { - return normalizePath(`${this.cacheDir}/${CacheManager.createCacheKey(request)}${extension.length > 0 ? `.${extension}` : ``}`); + return normalizePath(`${this.cacheDir}/${CacheManager.createCacheKeyFromRequest(request)}${extension.length > 0 ? `.${extension}` : ``}`); } - private filePathToCachedFileFromMetadata(metadata: CacheMetadata) { - return normalizePath(`${this.cacheDir}/${this.nameOfCachedFileFromMetadata(metadata)}`); + public filePathToCachedFileFromMetadata(metadata: CacheMetadata, cacheKey?: string) { + return normalizePath(`${this.cacheDir}/${this.nameOfCachedFileFromMetadata(metadata, cacheKey)}`); } - private nameOfCachedFileFromMetadata(metadata: CacheMetadata) { - return `${GetCacheKey(metadata.hash)}${metadata.file.ext.length > 0 ? `.${metadata.file.ext}` : ``}`; + /** + * + * @param metadata + * @param cacheKey Supply the cache key if you have it to avoid generating it again. + * @returns + */ + private nameOfCachedFileFromMetadata(metadata: CacheMetadata, cacheKey?: string) { + return `${cacheKey ?? CacheManager.createCacheKeyFromMetadata(metadata)}${metadata.f.e.length > 0 ? `.${metadata.f.e}` : ``}`; + } + + private fileInfoFromMetadata(metadata: CacheMetadata): FileInfo { + return { filename: metadata.f.n, extension: metadata.f.e }; } /** * 1. Downloads the {@link request} * 2. Creates its {@link CacheMetadata|metadata} - * 3. Writes files to disk. Tries to roll back changes on failure. + * 3. Writes downloaded item to disk. Tries to roll back changes on failure. * * @param request * @param fileInfo - * @returns `null` if any + * @returns */ - private async download(request: CacheRequest, fileInfo: FileInfo): Promise { + private async download(request: CacheRequest, fileInfo: FileInfo): Promise { - let metadata: CacheMetadata | null = null; - const sourceUrl = request.key; - const cacheKey = CacheManager.createCacheKey(request); + let result: CacheResult; + const sourceUrl = request.source; + const cacheKey = CacheManager.createCacheKeyFromRequest(request); try { - Log(`CacheManager:download: Requesting URL\n\t...${sourceUrl.slice(-50)}\n\tcacheID: ${cacheKey}`); - - const response = await requestUrl({ url: sourceUrl, method: 'GET', throw: true }); - const headers = Url.normalizeHeaders(response.headers); - const contentType: string | undefined = headers[Url.RESPONSE_HEADER_LOWERCASE.contentType]; + Log(`CacheManager:download: Requesting ${cacheKey} ⬇️⬇️⬇️\n\t...${sourceUrl.slice(-50)}`); if (ENV.dev) - await sleep(Math.floor(Math.random() * 1001) + 2000); - Log(`CacheManager:download: Got response:\n\tcacheID: ${cacheKey}\n\t${response.status}\n\tcontentType: ${contentType}`); + await sleep(Math.floor(Math.random() * 1001) + 1000); - if (!contentType) { - Log(`\tNO CONTENT-TYPE\n\t${sourceUrl}`) - console.log(response.headers); - } + const response = await requestUrl({ url: sourceUrl, method: 'GET', throw: false }); + const headers = Url.normalizeHeaders(response.headers); + const contentType = headers[Url.RESPONSE_HEADER_LOWERCASE.contentType] ?? undefined; + const cacheControl = headers[Url.RESPONSE_HEADER_LOWERCASE.cacheControl] ?? undefined; + + //Log(`CacheManager:download: Got response:\n\tcacheID: ${cacheKey}\n\t${response.status}\n\tcontentType: ${contentType}`); + + if (cacheControl == Url.CACHE_CONTROL_LOWERCASE.noStore) + throw new CacheError(`Caching not allowed on ${sourceUrl}.`, cacheKey); + + if (response.status >= 400) + throw new CacheFetchError(cacheKey, null, { sourceUrl: sourceUrl, statusCode: response.status }); const bytes = new Uint8Array(response.arrayBuffer); + const imageMetadata = this.handleImage(bytes); // Will throw if what was downloaded isn't a supported image type. This is checked before the file is written. const nowDateString = new Date().toISOString(); + const cacheItemPath = this.filePathToCachedFile(request, fileInfo.extension); - // Write file. try { - await this.vault.adapter.writeBinary(this.filePathToCachedFile(request, fileInfo.extension), response.arrayBuffer); - } catch (error) { - throw error; + await this.vault.adapter.writeBinary(cacheItemPath, response.arrayBuffer); + } + catch (writeError) { + try { + await this.vault.adapter.remove(cacheItemPath); + } catch (removeError) { + if (ENV.debugLog) + console.error("Failed to remove cached file after write error:", removeError); + } + throw writeError; } - // Create metadata and write file. + const metadata = { + ty: CacheType.IMAGE, + ti: { + d: nowDateString, + l: nowDateString, + cc: cacheControl, + }, + f: { + s: sourceUrl, + n: fileInfo.filename, + e: fileInfo.extension, + sz: response.arrayBuffer.byteLength, + ct: contentType, + ch: CacheManager.hashBinary(bytes), + }, + i: imageMetadata + }; + try { + this.metadataRoot.items[cacheKey] = metadata + await this.saveMetadata(); // Throws + result = new CacheResult(request, cacheKey, this.createCacheItem(metadata, false), null, true); - metadata = { - type: CacheType.IMAGE, - time: { - download: nowDateString, - lastAccess: nowDateString, - }, - retainer: { - //[`${request.filePath}`]: {} // TODO: Multiple Files Waiting Requesting the Same Cache - }, - file: { - src: sourceUrl, - name: fileInfo.filename, - ext: fileInfo.extension, - size: response.arrayBuffer.byteLength, - ct: contentType, - }, - hash: { - key: "keyMD5", - keyMD5: cacheKey, - cntMD5: CacheManager.HashMD5(bytes) - }, - image: this.handleImage(bytes) ?? undefined - }; - - await this.vault.adapter.write(this.filePathToAssociatedMetadata(request), JSON.stringify(metadata, null, 2)); + Log(`CacheManager:download\n\tDownloaded and cached ${this.nameOfCachedFileFromMetadata(metadata, cacheKey)}`); } catch (error) { - // The cache file was created but not its metadata. Reset and abort. - metadata = null; - await this.vault.adapter.remove(this.filePathToCachedFile(request, fileInfo.extension)); + + if (ENV.debugLog) + console.error("Failed to write cache metadata:", error); + + // File write failed. Rollback the cache and remove the potentially created file. + delete this.cache[cacheKey]; + + try { + await this.vault.adapter.remove(cacheItemPath); + } catch (removeError) { + if (ENV.debugLog) + console.error("Failed to remove cached file after write error:", removeError); + } + throw error; } - } catch (error) { - console.error(`Error downloading and caching image: ${error}`); + if (error instanceof CacheError) + result = new CacheResult(request, cacheKey, null, error); + else + result = new CacheResult(request, cacheKey, null, new CacheFetchError(cacheKey, error, { sourceUrl: sourceUrl })); } - return metadata; + return result; } - private handleImage(byteArray: Uint8Array): CacheMetadataImage | null { + /** + * Parses image metadata such as width and height. + * + * @param byteArray The image data as a `Uint8Array`. + * @returns The extracted image metadata. + * @throws {CacheTypeError} If the image type is unsupported. + * @throws {CacheError} If another error occurs while reading the image. + */ + private handleImage(byteArray: Uint8Array): CacheMetadataImage { - let sizeCalcResult: ISizeCalculationResult | undefined; - try { - // trows if unsupported file type, or other error. - sizeCalcResult = imageSize(byteArray); + try { + const sizeCalcResult = imageSize(byteArray); + + // https://github.com/image-size/image-size#jpeg-image-orientation + return { + w: sizeCalcResult.width, + h: sizeCalcResult.height, + t: sizeCalcResult?.type ?? "", + }; } catch (error) { - console.error(`Unsupported image type: ${error.message}`); + if (error instanceof TypeError) + throw new CacheTypeError(error); + else + throw new CacheError("", "Failed to read image.", { cause: error }); } - - // https://github.com/image-size/image-size#jpeg-image-orientation - - return sizeCalcResult ? { - width: sizeCalcResult.width, - height: sizeCalcResult.height, - type: sizeCalcResult?.type ?? "", - } : null; } - - private static HashMD5(array: Uint8Array) { - //Log(`MD5`); - return new Md5().appendByteArray(array).end() as string; - } - - // https://www.npmjs.com/package/xxhash-addon - // private static HashXXHash3(arrayBuffer: ArrayBuffer) { - // Log(`XXHash3`); - // return XXHash3.hash(Buffer.from(arrayBuffer)).toString('hex'); - // } } diff --git a/src/CacheMetadata.ts b/src/CacheMetadata.ts index 4557c27..3bc1acc 100644 --- a/src/CacheMetadata.ts +++ b/src/CacheMetadata.ts @@ -1,51 +1,11 @@ - -/** - * - All datetimes are stored in ISO 8601, e.g., "2023-10-27T10:30:00Z". - */ -export interface CacheMetadata { - type: CacheType; - file: CacheMetadataFile; - hash: CacheMetadataHash; - image?: CacheMetadataImage; - retainer: { [key: string]: CacheRetainer }; - time: CacheMetadataTime; +export interface CacheRoot { + retainers: Record + items: Record; } -export const enum CacheType { - UNDEFINED = 0, - IMAGE = 1, -}; - -export interface CacheMetadataFile { - src: string; - name: string; - /** If empty string, then no extension. */ - ext: string; - size: number; - - /** - * Content-Type form HTTP header. - */ - ct?: string; -} - -export interface CacheMetadataHash { - /** Which has to use as the key. Set to "keyMD5". */ - key: string; - /** The hash that is used as a key to identify each cache item. The cached files and their metadata also have this as their filename. */ - keyMD5: string; - cntMD5: string; -} - -export function GetCacheKey(hash: CacheMetadataHash) { - return hash[hash.key as keyof CacheMetadataHash]; -} - -export interface CacheMetadataImage { - width: number; - height: number; - /** Type as parsed from image data by [image-size](https://github.com/image-size/image-size). If empty string, then not determined. */ - type: string; +export const EMPTY_CACHE_ROOT: CacheRoot = { + retainers: {}, + items: {}, } /** @@ -53,10 +13,71 @@ export interface CacheMetadataImage { * The cache should only be remove when there are no "retainers". */ export interface CacheRetainer { - + /** Cache items referenced, i.e., retained. */ + ref: string[]; } +/** + * - All datetimes are stored in ISO 8601, e.g., "2023-10-27T10:30:00Z". + */ +export interface CacheMetadata { + ty: CacheType; + + f: CacheMetadataFile; + + i?: CacheMetadataImage; + + ti: CacheMetadataTime; +} + +export const enum CacheType { + UNDEFINED = 0, + IMAGE = 1, +}; + +/** Common things about a file and its content. */ +export interface CacheMetadataFile { + /** src */ + s: string; + + /** name */ + n: string; + + /** + * ext + * If empty string, then no extension. + */ + e: string; + + /** size in bytes */ + sz: number; + + /** + * Content-Type from the HTTP response header. + */ + ct?: string; + + /** xxHash of the file's content. */ + ch: string; +} + +/** Image specific */ +export interface CacheMetadataImage { + w: number; + h: number; + /** Type as parsed from image data by [image-size](https://github.com/image-size/image-size). If empty string, then not determined. */ + t: string; +} + +/** Time related */ export interface CacheMetadataTime { - download: string; - lastAccess: string; + + /** Download time */ + d: string; + + /** Last accessed */ + l: string; + + /** Value of the `Cache-Control` HTTP response header. */ + cc?: string; } \ No newline at end of file diff --git a/src/Environment.ts b/src/Environment.ts index bfdd940..81576b3 100644 --- a/src/Environment.ts +++ b/src/Environment.ts @@ -1,4 +1,4 @@ -import { Notice as ObsidianNotice } from "obsidian"; +import { App, Notice as ObsidianNotice, Platform } from "obsidian"; const isProduction = process.env.NODE_ENV === "production"; @@ -10,13 +10,13 @@ export const ENV = { export const Log = ENV.debugLog ? (msg: string, error?: Error): void => { - if (error) { - console.log(msg, error); - } else { - console.log(msg); - } + if (error) { + console.log(msg, error); + } else { + console.log(msg); } - : (): void => {}; + } + : (): void => { }; export class Notice { private innerNotice: ObsidianNotice; @@ -30,3 +30,18 @@ export class Notice { this.innerNotice = new ObsidianNotice(`${omit ? "" : `${Notice.NAME}: `}${msg}`, duration); } } + +export function clearBrowserCache(appOrCallback: (() => void) | App) { + if (ENV.dev && Platform.isDesktopApp) { + require('electron').remote.session.defaultSession.clearCache() + .then(() => { + if (appOrCallback instanceof App) { + // @ts-expect-error + app.commands.executeCommandById("app:reload"); + } + else + appOrCallback(); + }) + .catch((error: any) => console.error('Error clearing cache:', error)); + } +} diff --git a/src/HtmlAssistant.ts b/src/HtmlAssistant.ts index 3b42f0a..78c5e20 100644 --- a/src/HtmlAssistant.ts +++ b/src/HtmlAssistant.ts @@ -1,28 +1,68 @@ -import { ENV, Log } from "./Environment"; +import { ENV, Log } from "Environment"; +import { Url } from "Url"; export const enum HTMLElementCacheState { /** Untouched by the plugin. */ ORIGINAL = 0, - /** The element has been seen. The original src/url has been removed. */ - ORIGINAL_CANCELLED, - /** Waiting for cache. Note that the src attrib might be set to an SVG here. */ - CACHE_REQUESTED, + + /** + * The original src has been removed. Element is ready to request. + * + * A `data` attribute has been set on the element with the original src, get it with {@link HtmlAssistant.originalSrc}. + */ + ORIGINAL_SRC_REMOVED, + + /** + * Requesting cache. If cache found, will be changed to {@link CACHE_SUCCEEDED}; if not, to {@link REQUESTING_DOWNLOADING}. + */ + REQUESTING, + + /** + * Cache item was not found. Downloading. + */ + REQUESTING_DOWNLOADING, + + /** + * The element's src is now pointing to the locally cached item, whether it was fetched from cache or downloaded. + */ CACHE_SUCCEEDED, - /** Cache was requested but failed. Note that the src attrib might be set to an SVG here. */ + + /** + * Cache was requested but failed. As opposed to {@link INVALID}, these can be retried. + * Basically equal to {@link ORIGINAL_SRC_REMOVED} except that they have requested at least once. + * + * For example, connection failed. + */ CACHE_FAILED, + + /** + * For example, the url doesn't exist (404). + */ + INVALID, } -export enum HTMLElementAttribute { +export const enum HTMLElementAttribute { SRC = "src", ALT = "alt", -} +}; export class HtmlAssistant { - public static isCacheState(element: HTMLElement, state: HTMLElementCacheState): boolean { - return this.cacheState(element) == state; + public static isElementCacheStateEqual(element: HTMLElement, states: HTMLElementCacheState[]): boolean { + return this.isCacheStateEqual(this.cacheState(element), states); } + public static isCacheStateEqual(state: HTMLElementCacheState, states: HTMLElementCacheState[]): boolean { + for (const stateToCheck of states) + if (stateToCheck == state) + return true; + return false; + } + + /** + * @param element + * @returns If {@link element} is lacking a state, {@link HTMLElementCacheState.ORIGINAL} is returned. + */ public static cacheState(element: HTMLElement): HTMLElementCacheState { const state = element.dataset.comeDownState; @@ -33,10 +73,12 @@ export class HtmlAssistant { if ( numericState !== HTMLElementCacheState.ORIGINAL && - numericState !== HTMLElementCacheState.ORIGINAL_CANCELLED && - numericState !== HTMLElementCacheState.CACHE_REQUESTED && + numericState !== HTMLElementCacheState.ORIGINAL_SRC_REMOVED && + numericState !== HTMLElementCacheState.REQUESTING && + numericState !== HTMLElementCacheState.REQUESTING_DOWNLOADING && numericState !== HTMLElementCacheState.CACHE_SUCCEEDED && - numericState !== HTMLElementCacheState.CACHE_FAILED + numericState !== HTMLElementCacheState.CACHE_FAILED && + numericState !== HTMLElementCacheState.INVALID ) { throw new Error(`Invalid cache state: ${state}`); } @@ -48,64 +90,104 @@ export class HtmlAssistant { element.dataset.comeDownState = state.toString(); } - public static setSuccess(element: HTMLElement, src: string) { - HtmlAssistant.setCacheState(element, HTMLElementCacheState.CACHE_SUCCEEDED) - if (element instanceof HTMLImageElement) - element.setAttribute(HTMLElementAttribute.SRC, src); + /** + * Methods returns before the images have loaded but guarantees that they will load + * unless the returned result's `error` is set. + * + * @param imageElements + * @param src + * @returns Assume file not found when an {@link Error} is returned, see {@link createBlobObjectUrl}. + */ + public static async loadImages(imageElements: HTMLImageElement[], src: string) { + const result = await this.createBlobObjectUrl(src); + + if (result instanceof Error) { + return { + error: result, + fileNotFound: result instanceof HtmlAssistantFileNotFoundError, + }; + } + else { + let remainingLoads = imageElements.length; + + const onLoad = (event: Event) => { + const img = event.target as HTMLImageElement; + img.removeEventListener("load", onLoad); + + remainingLoads--; + if (remainingLoads === 0) { + URL.revokeObjectURL(result); + } + }; + + for (const imageElement of imageElements) { + imageElement.addEventListener("load", onLoad); + imageElement.setAttribute(HTMLElementAttribute.SRC, result); + } + + return null; + } + } + + /** + * Removes all the data sets from the element. + * @param element + */ + public static resetElement(element: HTMLElement) { + delete element.dataset.comeDownOriginalSource; + delete element.dataset.comeDownState; } public static setFailed(element: HTMLElement) { this.setCacheState(element, HTMLElementCacheState.CACHE_FAILED); if (element instanceof HTMLImageElement) - this.setIcon(element, HtmlAssistant.ENCODED_FAILED_ICON); + this.setIcon(element, this.SVG_FAILED_ICON_BLOB); + } + + public static setInvalid(element: HTMLElement) { + this.setCacheState(element, HTMLElementCacheState.INVALID); + if (element instanceof HTMLImageElement) + this.setIcon(element, this.SVG_FAILED_ICON_BLOB); } /** - * Removes the `src` attribute to prevent ordinary loading. - * Use {@link imageElementOriginalSrc} to get it `src` when needed. - * @param imageElements - * @returns The {@link imageElements} passed in for chaining. + * - Removes the `src` attribute to prevent ordinary loading. + * - Sets the state to {@link HTMLElementCacheState.ORIGINAL_SRC_REMOVED}. + * - Will not do anything if: `src` attrib is missing or empty string; if the original src already has been set to the data set. + * + * @param imageElements */ - public static preventImageLoading(imageElements: HTMLImageElement[]): HTMLImageElement[] { - Log(`preventImageLoading: ${imageElements.length} image elements`); + public static cancelImageLoading(imageElements: HTMLImageElement[]) { - const preventOnImage = (imageElement: HTMLImageElement): boolean => { - if (!imageElement.hasAttribute(HTMLElementAttribute.SRC)) + const cancelImageLoad = (imageElement: HTMLImageElement): boolean => { + + // Only continue if there's something to cancel. + const src = imageElement.getAttribute(HTMLElementAttribute.SRC); + if (!src) // Empty string is falsy. return false; - // TODO: Setting src="" immediately results in its value set to `app://obsidian.md/index.html` - if (imageElement.dataset.comeDownOriginalSource) { - Log(`\tAlready prevented. Skipping.`) - return false; + if (ENV.debugLog && src && Url.isBlob(src)) { + console.warn(`cancelImageLoading: Setting dataset to blob.`); } - const src = imageElement.src; - - if (src == "") { - delete imageElement.dataset.comeDownOriginalSource; - return false; - } - - HtmlAssistant.setCacheState(imageElement, HTMLElementCacheState.ORIGINAL_CANCELLED); // TODO: Neeed? As dataset will be set at the same time imageElement.dataset.comeDownOriginalSource = src; - - // TODO: Setting src="" immediately results in its value set to `app://obsidian.md/index.html` - //imageElement.src = ""; - - //Log(`\tRemoving src="...${imageElement.src.slice(-50)}"`); imageElement.removeAttribute(HTMLElementAttribute.SRC); + HtmlAssistant.setCacheState(imageElement, HTMLElementCacheState.ORIGINAL_SRC_REMOVED); return true; }; + let counter = 0; imageElements.forEach((imageElement) => { - //Log(`\tsrc: ${imageElement.src}, len:${imageElement.src.length}`); - - if (!preventOnImage(imageElement) && ENV.dev && imageElement.hasAttribute(HTMLElementAttribute.SRC)) - Log(`\tSrc: ${imageElement.src}, len:${imageElement.src.length}`); + if (cancelImageLoad(imageElement)) + counter++; }); - return imageElements; + Log(`cancelImageLoading: Cancelled ${counter} of ${imageElements.length}.`); + } + + public static originalSrc(element: HTMLImageElement) { + return element.dataset.comeDownOriginalSource ?? null; } /** @@ -142,73 +224,29 @@ export class HtmlAssistant { return null;*/ } - /** - * 1. Finds all image elements that are of interest - * 2. Discards those already known. - * 3. Calls {@link preventImageLoading} on the remaining. - * - * @param element Element to start searching from. - * @returns - */ - public static preparedImageElements(imageElements: HTMLImageElement[]): HTMLImageElement[] { - if (imageElements.length == 0) - return imageElements; - - Log(`preparedImageElements`); - - let processed = 0; - let original = 0; - - const filtered = imageElements.filter((imageElement) => { - switch (this.cacheState(imageElement)) { - // These are new - case HTMLElementCacheState.ORIGINAL: { - //Log(`\tState ORIGINAL`); - //Log(`\t\t${imageElement.outerHTML}`); - original++; - return true; - } - - // These have passed this stage - case HTMLElementCacheState.ORIGINAL_CANCELLED: - case HTMLElementCacheState.CACHE_REQUESTED: - case HTMLElementCacheState.CACHE_SUCCEEDED: { - processed++; - return false; - } - - // What TODO: - case HTMLElementCacheState.CACHE_FAILED: - return false; - - default: - throw new Error(`Unhandled cache state.`); - } - }); - - Log(`\tTotal: ${imageElements.length}, original: ${original}, processed: ${processed}`); - - return filtered.length > 0 ? HtmlAssistant.preventImageLoading(filtered) : filtered; - } /** - * - * - 'img' selects all elements. - * - [src] selects only elements that have a src attribute. - * - :not([aria-hidden="true"]) filters out elements that have the aria-hidden="true" attribute. - * - * @param element Root element - * @param requireSrcAttribute - * @returns - */ - public static findAllRelevantImages(element: HTMLElement, requireSrcAttribute: boolean = true): HTMLImageElement[] { + * This method is intended for finding unprocessed elements. + * + * - 'img' selects all elements. + * - [src] selects only elements that have a src attribute. + * - :not([aria-hidden="true"]) filters out elements that have the aria-hidden="true" attribute. + * + * @param element + * @param requireSrcAttribute If `true` will not return image elements that don't have the `src` attribute. + * @param filter + * @returns + */ + public static findRelevantImagesToProcess(element: HTMLElement, requireSrcAttribute: boolean = true, filter?: (imageElement: HTMLImageElement) => boolean): HTMLImageElement[] { + let imageElements; if (requireSrcAttribute) - return element.findAll('img[src]:not([aria-hidden="true"])') as HTMLImageElement[]; + imageElements = element.findAll('img[src]:not([aria-hidden="true"])') as HTMLImageElement[]; else - return element.findAll('img:not([aria-hidden="true"])') as HTMLImageElement[]; + imageElements = element.findAll('img:not([aria-hidden="true"])') as HTMLImageElement[]; - /* - Gave up with this. Not sure if it's more efficient than view.contentDOM.findAll. + return filter ? imageElements.filter((imageElement) => filter(imageElement)) : imageElements; + + /* syntaxTree(view.state).iterate({ enter: ({ type, from, to }: SyntaxNodeRef) => { console.log(`${view.state.doc.sliceString(from, to)}`); @@ -217,39 +255,85 @@ export class HtmlAssistant { */ } - public static setIcon(imageElement: HTMLImageElement, engodedSvg: string) { - imageElement.setAttribute(HTMLElementAttribute.SRC, `data:image/svg+xml;charset=utf-8,${engodedSvg}`); + /** + * + * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/File_API/Using_files_from_web_applications#using_object_urls|Using object URLs} + * + * @remarks + * + * If {@link src} points to a local file that doesn't exist, the Dev Tools will show "net::ERR_FILE_NOT_FOUND". + * The net::ERR_FILE_NOT_FOUND message you see in the console comes from the browser’s internal networking layer, only shown in DevTools, not available to JavaScript code. + * However, in Electron and Chrome-based browsers, fetch() throwing a TypeError generally means the file doesn’t exist. + * + * Capacitor also, which runs on the native mobile webview, also throws a `TypeError` on iOS (not tested on Android). + * + * To be safe, assume all `Error`s is file not found. + * + * @param src Url to a local resource. Would start with `app://`, `capacitor://`, or perhaps `file://` + * @returns On failure, a {@link HtmlAssistantFileNotFoundError} if it thinks the a give local file doesn't exist; otherwise a normal Error. + */ + public static async createBlobObjectUrl(src: string): Promise { + let response; + + try { + // Response from `requestUrl` does not allow for creating blobs. These will always be local urls, so no need to handle CORS. + response = await fetch(src); + } catch (error) { + if (error instanceof TypeError) + return new HtmlAssistantFileNotFoundError(src); + else + return error; + } + + if (response.ok) { + try { + const blob = await response.blob() + const url = URL.createObjectURL(blob); + if (url && URL.canParse(url) && Url.isBlob("blob:")) + return url; + else + return new Error(`Invalid blob URL: ${url}`); + } catch (error) { + return error; + } + } + else { + return new Error(`Unsuccessful response: ${response.status} ${response.statusText}`); + } + } + + public static setLoadingIcon(imageElement: HTMLImageElement) { + this.setIcon(imageElement, this.SVG_LOADING_ICON_BLOB); + } + + private static setIcon(imageElement: HTMLImageElement, blob: Blob) { + const url = URL.createObjectURL(blob); + const onLoad = (_event: Event) => { + imageElement.removeEventListener("load", onLoad); + URL.revokeObjectURL(url); + } + imageElement.addEventListener("load", onLoad); + imageElement.setAttribute(HTMLElementAttribute.SRC, url); } /** - * @todo - * @see {@link https://lucide.dev/icons/loader} - */ - private static readonly SVG_LOADING_ICON = ` - - - - - - - - - - - `; + * @todo + * @see {@link https://lucide.dev/icons/loader} + */ + private static readonly SVG_LOADING_ICON_BLOB = new Blob( + [``], + { type: "image/svg+xml" } + ); - private static readonly SVG_FAILED_ICON = ``; + private static readonly SVG_FAILED_ICON_BLOB = new Blob( + [``], + { type: "image/svg+xml" } + ); +} - static readonly ENCODED_LOADING_ICON = encodeURIComponent(this.SVG_LOADING_ICON); - static readonly ENCODED_FAILED_ICON = encodeURIComponent(this.SVG_FAILED_ICON); +class HtmlAssistantFileNotFoundError extends Error { + constructor(url: string) { + super(`File not found: ${url}`); + this.name = "HtmlAssistantFileNotFoundError"; + } } \ No newline at end of file diff --git a/src/InfoModal.ts b/src/InfoModal.ts new file mode 100644 index 0000000..f5b5d53 --- /dev/null +++ b/src/InfoModal.ts @@ -0,0 +1,100 @@ +import { App, ButtonComponent, Modal, Setting } from 'obsidian'; +import { CacheManager } from 'CacheManager'; +import { PluginSettings } from 'Settings'; +import { Notice } from 'Environment'; + +export class InfoModal extends Modal { + private cacheManager: CacheManager; + private settings: PluginSettings; + + constructor(app: App, cacheManager: CacheManager, settings: PluginSettings) { + super(app); + this.cacheManager = cacheManager; + this.settings = settings; + + this.setTitle('Cacheboard'); + + this.clearCacheSetting = new Setting(this.contentEl) + .setName('Total number of files cached') + .addButton((button) => { + this.clearCacheButton = button; + button.buttonEl.tabIndex = -1; + button.setButtonText("Delete"); + button.onClick(() => { + button.setButtonText("Confirm cache delete"); + button.setWarning(); + button.onClick(async () => { + + await cacheManager.clearCached((error) => { + if (error) { + new Notice(`An error occured while clearing the cache: ${error.message}`, 0); + console.error("Error clearing cache.", error); + } + else { + new Notice("Cache cleared"); + + button.buttonEl.remove(); + this.populate(); + } + }); + }); + }); + }); + + if (settings.showDebugInfo) { + this.debugInfoSetting = new Setting(this.contentEl) + .setName('Debug info') + .addButton((button) => { + button.setButtonText("Refresh metadata"); + button.onClick(async () => { + await this.cacheManager.debug().loadMetadata(); + this.populate(); + }); + }); + } + + new Setting(this.contentEl) + .addButton((button) => { + this.closeButton = button; + button.buttonEl.tabIndex = 1; + button.setButtonText("Close"); + button.onClick(() => { + this.close(); + }); + }); + } + + debugInfoSetting?: Setting; + clearCacheSetting: Setting; + clearCacheButton: ButtonComponent; + closeButton: ButtonComponent; + + onOpen(): void { + super.onOpen(); + + setTimeout(() => { + this.clearCacheButton.buttonEl.tabIndex = 0 + this.closeButton.buttonEl.focus(); + }); + + this.populate(); + } + + private populate() { + setTimeout(() => { + + this.debugInfoSetting?.descEl.empty(); + + this.cacheManager.info((info) => { + this.clearCacheSetting.setDesc(`${info.numberOfFilesCached} file${info.numberOfFilesCached == 1 ? `` : `s`}`); + + if (this.debugInfoSetting) { + for (const line of info.summary.split("\n")) + this.debugInfoSetting.descEl.createEl("div", {}, (p) => { + p.innerText = line; + }) + } + }); + }); + } +} \ No newline at end of file diff --git a/src/ProcessingPass.ts b/src/ProcessingPass.ts new file mode 100644 index 0000000..9751d2e --- /dev/null +++ b/src/ProcessingPass.ts @@ -0,0 +1,159 @@ +import { EditorView, ViewUpdate } from "@codemirror/view"; +import { MarkdownView, MarkdownPostProcessorContext, FileView, TFile, App } from "obsidian"; +import { Log, ENV } from "Environment"; +import { CacheRequest, CacheManager } from "CacheManager"; + +/** + * - Gathers some state useful during the current pass. + * - Call first in system callbacks to abort as early as possible. + */ +export class ProcessingPass { + + private markdownView: MarkdownView | null; + public associatedFile: TFile; + + /** `true` when invoked by the Markdown post processor. */ + public isInPostProcessingPass: boolean; + private viewUpdate?: ViewUpdate; + + private static updatePassID: number = 0; + private static postProcessorPassID: number = 0; + public passID: number; + + public static beginFromViewUpdate(app: App, viewUpdate: ViewUpdate, noFile: () => void, sourceMode?: () => void): ProcessingPass | null { + + const processPassID = this.updatePassID++; + + if (ENV.dev) { + + const updated = this.viewUpdateChanges(viewUpdate); + const changed = Object.keys(updated).filter(key => updated[key]); + const notChanged = Object.keys(updated).filter(key => !updated[key]); + + Log(`🥎 editorViewUpdateListener ${changed.length} view updates. 🛎️ ID ${processPassID}`); + if (changed.length > 0) + Log(`\t🟢 ${changed.join(", ")}`); + if (changed.length > 0 && notChanged.length > 0) + Log(`\t🔴 ${notChanged.join(", ")}`); + } + + const markdownView = app.workspace.getActiveViewOfType(MarkdownView); + const associatedFile = markdownView?.file ?? null; + + // `app.workspace.getActiveFile()` is of no help. + if (!associatedFile) { + Log(`\tNo associated file. ID ${processPassID}`); + noFile(); + return null; + } + else if (markdownView && this.isInSourceMode(markdownView)) { + sourceMode?.(); + return null; + } + else { + const instance = new this(); + instance.passID = processPassID; + instance.markdownView = markdownView; + instance.associatedFile = associatedFile; + instance.isInPostProcessingPass = false; + instance.viewUpdate = viewUpdate; + + return instance; + } + } + + public static beginFromPostProcessorContext(app: App, context: MarkdownPostProcessorContext) : ProcessingPass { + + const postProcessorPassID = this.postProcessorPassID++; + + Log(`🏀 postProcessReadingModeHtml 🛎️ ID ${postProcessorPassID}`); + + const instance = new this(); + instance.passID = postProcessorPassID; + instance.isInPostProcessingPass = true; + + instance.markdownView = app.workspace.getActiveViewOfType(MarkdownView); + + let associatedFile = instance.markdownView?.file ?? app.vault.getFileByPath(context.sourcePath); + console.assert(associatedFile); + instance.associatedFile = associatedFile!; + + return instance; + } + + private static viewUpdateChanges(viewUpdate: ViewUpdate) { + const u: Record = {}; + + u["docChanged"] = viewUpdate.docChanged; + u["viewportChanged"] = viewUpdate.viewportChanged; + u["selectionSet"] = viewUpdate.selectionSet; + u["focusChanged"] = viewUpdate.focusChanged; + u["geometryChanged"] = viewUpdate.geometryChanged; + u["heightChanged"] = viewUpdate.heightChanged; + u["viewportMoved"] = viewUpdate.viewportMoved; + + return u; + } + + private static isInLivePreviewFromView(view: FileView) { + const state = view?.getState(); + return state ? state.mode == "source" && state.source == false : false; + } + + private static isInSourceMode(view: FileView) { + const state = view?.getState(); + return state ? state.mode == "source" && state.source == true : false; + } + + /** + * The {@link EditorView.updateListener.of} {@link Extension} is called in both reader and preview/source mode and let's you process all elements at once. + * The {@link registerMarkdownPostProcessor} is only called first when the file is opened in reader mode. It is called several times as the DOM is beeing built, which makes it complicated to consolidate them into one call. + * + * Because the update listener extension callback supplies the DOM of the whole file's writing area, and further, because it is always run before the Markdown post processor, + * it is enough and simplest to show the download notice only when invoked from the extension callback, i.e., when {@link processingContext.isInPostProcessingPass} is `true`. + * + * + * @param processingContext + * @param run + * @returns + */ + public runInUpdatePass(run?: () => void) { + if (this.isInUpdatePass) { + run?.(); + return true; + } + return false; + } + + public get isInUpdatePass() { + return this.viewUpdate ? true : false; + } + + public retainCacheFromRequest(request: CacheRequest) { + console.assert(request.requesterPath == this.associatedFile.path); + if (this.isInUpdatePass) + this.requestsToRetain.push(request); + } + + public get currentNumberOfRequestsToRetain() { + return this.requestsToRetain.length; + } + + private requestsToRetain: CacheRequest[] = []; + + public end(cacheManager: CacheManager) { + Log(`${this.isInUpdatePass ? `🥎 ProcessingPass:end: Update pass ✅ ID ${this.passID}` : `🏀 ProcessingPass:end: Post processing pass ✅ ID ${this.passID}`}`); + + if (this.isInUpdatePass) { + cacheManager.updateRetainedCaches(this.requestsToRetain, this.associatedFile.path).then(() => { + Log(`\tSaving metadata: ID ${this.passID}`); + cacheManager.saveMetadataIfDirty(); + }); + } + } + + public abort() { + Log(`${this.isInUpdatePass ? `🥎 ProcessingPass:abort: Update pass ⛔ ID ${this.passID}` : `🏀 ProcessingPass:end: Post processing pass ⛔ ID ${this.passID}`}`); + } + +} diff --git a/src/Settings.ts b/src/Settings.ts index a134f36..b03ff0a 100644 --- a/src/Settings.ts +++ b/src/Settings.ts @@ -1,10 +1,12 @@ -import { PluginSettingTab, Setting, Plugin } from "obsidian"; +import { PluginSettingTab, Setting, Plugin, Platform } from "obsidian"; +import { ENV, Notice } from "Environment"; +import { CacheManager } from "CacheManager"; export interface PluginSettings { - /** Shows a {@link Notice} when file download starts. */ + /** Show a {@link Notice} when file download starts. */ noticeOnDownload: boolean - + /** Remove the name of the plugin when showing the download message. */ omitNameInNotice: boolean; @@ -15,24 +17,27 @@ export interface PluginSettings { * This can't be disabled in the UI. But those who take matters into their own hands with data.json... */ gitIgnoreCacheDir: boolean; + + showDebugInfo: boolean; } export class SettingsManager { - settings: PluginSettings; - save: () => Promise; - onChangedCallback: (name: string) => void | undefined; + public settings: PluginSettings; + public save: () => Promise; + public onChangedCallback: (name: string, value: any) => void | undefined; static readonly DEFAULT_SETTINGS: PluginSettings = { noticeOnDownload: true, omitNameInNotice: false, gitIgnoreCacheDir: true, + showDebugInfo: false, } as const; static readonly SETTING_NAME = { gitIgnoreCacheDir: "gitIgnoreCacheDir", } as const; - constructor(settings: any, save: (settings: PluginSettings) => Promise, onChangedCallback: (name: string) => void | undefined) { + constructor(settings: any, save: (settings: PluginSettings) => Promise, onChangedCallback: (name: string, value: any) => void | undefined) { this.settings = settings; this.save = () => save(this.settings); this.onChangedCallback = onChangedCallback; @@ -40,11 +45,13 @@ export class SettingsManager { } export class SettingTab extends PluginSettingTab { - settingsManager: SettingsManager; + private settingsManager: SettingsManager; + private cacheManager: CacheManager; - constructor(plugin: Plugin, settingsManager: SettingsManager) { + constructor(plugin: Plugin, settingsManager: SettingsManager, cacheManager: CacheManager) { super(plugin.app, plugin); this.settingsManager = settingsManager; + this.cacheManager = cacheManager; } display(): void { @@ -53,6 +60,14 @@ export class SettingTab extends PluginSettingTab { containerEl.empty(); + let compactDownloadMessage: Setting | undefined; + const setCompactDownloadMessageVisibility = () => { + if (settings.noticeOnDownload) + compactDownloadMessage?.settingEl.show(); + else + compactDownloadMessage?.settingEl.hide(); + }; + new Setting(containerEl) .setName("Display a message on download") .setDesc("Let’s you know when files are downloaded and how many.") @@ -60,13 +75,14 @@ export class SettingTab extends PluginSettingTab { toggle.setValue(settings.noticeOnDownload); toggle.onChange(async (value) => { settings.noticeOnDownload = value; + setCompactDownloadMessageVisibility(); await this.settingsManager.save(); }); }); - new Setting(containerEl) - .setName("Do not show pluign’s name in download message") - .setDesc(`Toggle on to remove 'Come Down:' from the message.`) + compactDownloadMessage = new Setting(containerEl) + .setName("Use compact download message") + .setDesc("Shortens the message to make it less distracting.") .addToggle((toggle) => { toggle.setValue(settings.omitNameInNotice); toggle.onChange(async (value) => { @@ -74,7 +90,50 @@ export class SettingTab extends PluginSettingTab { await this.settingsManager.save(); }); }); - + setCompactDownloadMessageVisibility(); + + const cachedFilesSetting = new Setting(containerEl).setName('Number of files cached'); + setTimeout(() => { + this.cacheManager.actualCachedFilePaths().then((filePaths) => { + const num = filePaths.length; + cachedFilesSetting.setDesc(`${num} file${num == 1 ? `` : `s`}.`); + + if (num == 0) + return; + + cachedFilesSetting.addButton((button) => { + button.buttonEl.tabIndex = -1; + button.setButtonText("Delete all cached files"); + button.onClick(() => { + button.setButtonText("Confirm cache delete"); + button.setWarning(); + button.onClick(async () => { + + await this.cacheManager.clearCached((error) => { + if (error) { + new Notice(`An error occured while clearing the cache: ${error.message}`, 0); + console.error("Error clearing cache.", error); + } + else { + new Notice("Cache cleared"); + cachedFilesSetting.setDesc("Cache is empty.") + + button.buttonEl.remove(); + + if (ENV.dev && Platform.isDesktopApp) { + require('electron').remote.session.defaultSession.clearCache() + .then(() => new Notice('Electron Cache cleared successfully. Restart vault.')) + .catch((error: any) => console.error('Error clearing cache:', error)); + } + } + }); + }); + }); + }); + + }) + }); + new Setting(containerEl) .setName("Advanced").setHeading(); @@ -96,8 +155,8 @@ export class SettingTab extends PluginSettingTab { settings.gitIgnoreCacheDir = value; await this.settingsManager.save(); refreshDisabled(); - this.settingsManager?.onChangedCallback(SettingsManager.SETTING_NAME.gitIgnoreCacheDir); + this.settingsManager?.onChangedCallback(SettingsManager.SETTING_NAME.gitIgnoreCacheDir, value); }); }) } -} \ No newline at end of file +} diff --git a/src/Url.ts b/src/Url.ts index 94ae47f..988cc06 100644 --- a/src/Url.ts +++ b/src/Url.ts @@ -3,36 +3,41 @@ export class Url { /** - * These are case-sensitive. Use {@link normalizedHeaders} to be safer. + * These are case-sensitive. Use with {@link normalizedHeaders}. */ - public static readonly RESPONSE_HEADER = { - contentType: "Content-Type", - contentLength: "Content-Length", - cacheControl: "Cache-Control", - expires: "Expires", - }; - public static readonly RESPONSE_HEADER_LOWERCASE = { - contentType: Url.RESPONSE_HEADER.contentType.toLowerCase(), - contentLength: Url.RESPONSE_HEADER.contentLength.toLowerCase(), - cacheControl: Url.RESPONSE_HEADER.cacheControl.toLowerCase(), - expires: Url.RESPONSE_HEADER.expires.toLowerCase(), - }; + contentType: "Content-Type".toLowerCase(), + contentLength: "Content-Length".toLowerCase(), + cacheControl: "Cache-Control".toLowerCase(), + expires: "Expires".toLowerCase(), + } as const; + + public static readonly CACHE_CONTROL_LOWERCASE = { + noStore: "no-store", + } as const; /** * Normalize headers to make sure to, e.g., find both `Content-Type` and `content-type`. + * Also ignores empty strings and trims. */ public static normalizeHeaders(headers: Record): Record { const normalizedHeaders: Record = {}; + for (const key in headers) - normalizedHeaders[key.toLowerCase()] = headers[key]; + if (key.length > 0) + normalizedHeaders[key.toLowerCase().trim()] = headers[key]; + return normalizedHeaders; } - /** - * @author Gemini + public static isValid(url: string): boolean { + return url && URL.canParse(url) ? true : false; + } + + /** + * These are not relevant: ftp, mailto, tel, ws: and wss: */ - public static isExternal(src: string): boolean { + public static isExternal(src: string): boolean { try { const url = new URL(src); return url.protocol === "http:" || url.protocol === "https:"; @@ -41,12 +46,25 @@ export class Url { } } + public static isEmbedded(url: string): boolean { + return this.isBlob(url) || url.startsWith("data:"); + } + + public static isBlob(url: string): boolean { + return url.startsWith("blob:"); // Note: no slashes + } + + public static isLocal(url: string): boolean { + // Slashes are better: e.g., "app:data" or "file:info" are not URLs. + return url.startsWith("app://") || url.startsWith("capacitor://") || url.startsWith("file://"); + } + /** * @author Gemini * @param url * @returns */ - public static extractFilenameAndExtension(url: string): { filename: string, extension: string} | null { + public static extractFilenameAndExtension(url: string): { filename: string, extension: string } | null { try { const urlObj = new URL(url); const pathname = urlObj.pathname; diff --git a/src/main.ts b/src/main.ts index c7bad73..0ab56a7 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,16 +1,18 @@ -import { MarkdownView, Plugin, editorLivePreviewField, MarkdownPostProcessorContext, normalizePath, Platform, FileView, TFile } from "obsidian"; -import { SettingTab, SettingsManager, PluginSettings } from "src/Settings"; -import { CacheManager, CacheRequest } from "src/CacheManager"; import { EditorView, ViewUpdate } from "@codemirror/view"; -import { Log, Notice, ENV } from "src/Environment"; -import { HTMLElementAttribute, HTMLElementCacheState, HtmlAssistant } from "src/HtmlAssistant"; -import { GetCacheKey } from "./CacheMetadata"; +import { Plugin, MarkdownPostProcessorContext, normalizePath, TFile } from "obsidian"; +import { SettingTab, SettingsManager, PluginSettings } from "Settings"; +import { CacheFetchError, CacheItem, CacheManager, CacheRequest, CacheTypeError } from "CacheManager"; +import { Log, Notice, ENV, clearBrowserCache } from "Environment"; +import { HTMLElementAttribute, HTMLElementCacheState, HtmlAssistant } from "HtmlAssistant"; +import { InfoModal } from "InfoModal"; +import { ProcessingPass } from "ProcessingPass"; +import { Url } from "Url"; interface PluginData { settings: PluginSettings; } -export const DEFAULT_DATA: PluginData = { +const DEFAULT_DATA: PluginData = { settings: SettingsManager.DEFAULT_SETTINGS, } as const; @@ -21,94 +23,100 @@ export default class ComeDownPlugin extends Plugin { async onload() { + //#region Init + + //const startTime = performance.now(); + Notice.setName(this.manifest.name); this.data = Object.assign({}, DEFAULT_DATA, await this.loadData()); - //#region Settings + await this.ensureCacheDir(); + await this.ensureGitIgnore(); + + this.cacheManager = await CacheManager.create(this.app.vault, this.cacheDir, this.cacheMetadataPath, [this.gitIgnorePath]); this.settingsManager = new SettingsManager( this.data.settings, async (_settings) => await this.saveData(this.data), (name) => { if (name === SettingsManager.SETTING_NAME.gitIgnoreCacheDir) - this.ensureCacheDir(); + this.ensureGitIgnore(); } ); - this.addSettingTab(new SettingTab(this, this.settingsManager)); + this.addSettingTab(new SettingTab(this, this.settingsManager, this.cacheManager)); - //#endregion - - //#region Cache - - this.cacheManager = new CacheManager(this.app.vault, this.cacheDir); - this.ensureCacheDir(); + //console.log(`ComeDown: init: ${performance.now() - startTime} milliseconds`); //#endregion //#region Register - // The post processor runs after the Markdown has been processed into HTML. It lets you add, remove, or replace HTML elements to the rendered document. - this.registerMarkdownPostProcessor((e, c) => this.processReadingMode(e, c)); - + this.registerMarkdownPostProcessor((e, c) => this.postProcessReadingModeHtml(e, c)); this.registerEditorExtension(EditorView.updateListener.of((vu) => this.editorViewUpdateListener(vu))); - this.addCommand({ - id: "clear-all-cache", - name: "Clear All Cached Files", - callback: async () => { - await this.cacheManager.clearCached([this.gitIgnorePath], (error) => { - if (error) { - new Notice(`An error occured while clearing the cache: ${error.message}`, 0); - console.error(`Error clearing cache: ${error}`); - } - else { - new Notice(`Cache cleared.`); - if (ENV.dev && Platform.isDesktopApp) { - require('electron').remote.session.defaultSession.clearCache() - .then(() => { - new Notice('Electron Cache cleared successfully. Restart vault.'); - }) - .catch((error: any) => { - console.error('Error clearing cache:', error); - }); - } - } - }); - } - }); + this.registerEvent( + this.app.vault.on('delete', (file) => { + if (file instanceof TFile) { + this.cacheManager.removeRetainer(file.path).then(() => this.cacheManager.saveMetadataIfDirty()); + } + }) + ); - // this.addCommand({ - // id: 'redownload-all-images-in-this-file', - // name: 'Redownload all images in this file.', - // checkCallback: (checking: boolean) => { - // const markdownView = this.app.workspace.getActiveViewOfType(MarkdownView); - // if (markdownView) { - // if (!checking) { - // } - // return true; - // } - // } - // }); + this.registerEvent( + this.app.vault.on('rename', (file, oldPath) => { + if (file instanceof TFile) { + this.cacheManager.renameRetainer(oldPath, file.path); + this.cacheManager.saveMetadataIfDirty(); + } + }) + ); + + if (ENV.dev) { + + this.addCommand({ + id: "open-info-modal", + name: "Open Cacheboard", + callback: () => { + new InfoModal(this.app, this.cacheManager, this.settingsManager.settings).open(); + } + }); + + this.addCommand({ + id: "delete-all-cache-and-reload", + name: "Delete cache and reload", + callback: () => { + this.cacheManager.clearCached((error) => { + if (error) + console.error("Failed to clear cache", error); + else + clearBrowserCache(this.app); + }); + } + }); + } //#endregion + } async onunload() { await this.cacheManager?.cancelAllOngoing(); } - //#region Cache + //#region Cache Init /** * Will be set once it's ensured that `this.manifest.dir` is set. - * @throws {Error} - If this.manifest.dir isn't set. + * @throws {Error} If `this.manifest.dir` isn't set. */ get cacheDir(): string { if (this.cacheDirBacking === undefined) { const pluginDir = this.manifest.dir; if (pluginDir) { - this.cacheDirBacking = `${pluginDir}/cache`; + this.cacheDirBacking = normalizePath(`${pluginDir}/cache`); + this.gitIgnorePath = normalizePath(`${this.cacheDirBacking}/.gitignore`); + this.cacheMetadataPath = normalizePath(`${pluginDir}/cache.json`); } else { const errorMsg = `Cannot load because plugin directory is unknown`; @@ -118,24 +126,27 @@ export default class ComeDownPlugin extends Plugin { } return this.cacheDirBacking; } - private cacheDirBacking: string | undefined; - - get gitIgnorePath() { - if (!this.gitIgnorePathBacking) - this.gitIgnorePathBacking = normalizePath(`${this.cacheDir}/.gitignore`); - return this.gitIgnorePathBacking; - } - private gitIgnorePathBacking: string | undefined; + private cacheDirBacking?: string; + private gitIgnorePath: string; + private cacheMetadataPath: string; + /** + * Ensures the cache directory exists. + * + * Do not catch {@link Error}s so as to prevent the plugin from being enabled in such cases. + */ async ensureCacheDir() { - const ensureGitIgnore = this.settingsManager.settings.gitIgnoreCacheDir; - const cacheFolderExists = await this.app.vault.adapter.exists(this.cacheDir); if (!cacheFolderExists) await this.app.vault.adapter.mkdir(this.cacheDir); + } + /** + * Make sure a `.gitignore` file is added/removed. + */ + async ensureGitIgnore() { + const ensureGitIgnore = this.data.settings.gitIgnoreCacheDir; const gitignoreExists = await this.app.vault.adapter.exists(this.gitIgnorePath); - if (ensureGitIgnore && !gitignoreExists) await this.app.vault.adapter.write(this.gitIgnorePath, "*"); else if (!ensureGitIgnore && gitignoreExists) @@ -144,293 +155,305 @@ export default class ComeDownPlugin extends Plugin { //#endregion - //#region - - private isInLivePreviewFromView(view: FileView) { - const state = view?.getState(); - return state ? state.mode == "source" && state.source == false : false; - } - - private isInSourceModeFromView(view: FileView) { - const state = view?.getState(); - return state ? state.mode == "source" && state.source == true : false; - } - - private inLivePreviewFromEditorView(view: EditorView) { - return view.state.field(editorLivePreviewField); - } - - private getEditorRoot(element: HTMLElement): HTMLElement | null { - let currentElement: HTMLElement | null = element; - - while (currentElement) { - if (currentElement.classList.contains("view-content")) { - return currentElement; - } - currentElement = currentElement.parentElement as HTMLElement | null; - } - - return null; // Root not found - } + //#region Processing /** + * Remove requesting, downloading, done, and invalid. + * - The done image element is already pointing to the cached resource. No need to do anything more. + * - Those that are downloading will be handled as the download finishes as part of a previous pass. * - * @param view - * @returns true if file is open in Reading view. - */ - private isInReadingViewFromView(view: FileView): boolean { - return view?.getState().mode == "preview"; // "source" - } - - private viewModeString(fileView: FileView, editorView: EditorView | undefined = undefined) { - return this.isInReadingViewFromView(fileView) ? "Reading view (reading)" : "Live preview (editing)"; - //if (editorView) - // Log(`\t${!this.inLivePreviewFromEditorView(editorView) ? `Reading view (reading)` : `Live preview (editing)`}`) - } - - /** - * Call first in system callbacks to abort as early as possible. - * @param context - * @returns Do nothing further if `null`. - */ - private proceed(context?: MarkdownPostProcessorContext): { markdownView: MarkdownView, associatedFile: TFile } | null { - const markdownView = this.app.workspace.getActiveViewOfType(MarkdownView); - - if (!markdownView) { - Log(`\tSkipping because no active view.`); - return null; - } - - if (this.isInSourceModeFromView(markdownView)) { - Log(`\tSkipping because in source mode.`); - return null; - } - - // if (ENV.usePostProcessor && this.isInReadingViewFromView(markdownView)) { - // Log(`\tSkipping because in Reading view.`); - // return null; - // } - - const associatedFile = markdownView.file; // == this.app.workspace.getActiveFile() - if (!associatedFile) { - Log(`\tSkipping because in no associated file.`); - return null; - } - - if (context && associatedFile.path != context.sourcePath) { - console.warn(`\tExpected \`getActiveFile().path\` be equal to \`sourcePath\` of \`MarkdownPostProcessorContext\``) - return null; - } - - return { markdownView, associatedFile: associatedFile }; - } - - /** - * Finds the relevant html elements under {@link element} then aims to further filter out those that are unwanted. + * Keep + * - Original: these need to be cancelled + * - Cancelled and Failed: these need to request cache. * - * @param element + * @param imageElement * @returns */ - private imageElements(element: HTMLElement): HTMLImageElement[] { + private filterIrrelevantCacheStates(imageElement: HTMLImageElement) { - // Note that if the `src` attribute is used for an svg icon this method will regard it as relevant and return it. But it will be filtered when the state is checked. - let imageElements = HtmlAssistant.findAllRelevantImages(element); - - if (imageElements.length == 0) { - Log(`\tAborting: Found no relevant image elements.`) - return imageElements; - } - else - Log(`\tNumber of relevant image elements: ${imageElements.length}`); + const state = HtmlAssistant.cacheState(imageElement) - imageElements = imageElements.filter((imageElement) => { - const state = HtmlAssistant.cacheState(imageElement) - Log(`\tCache state: ${state}`); - return state != HTMLElementCacheState.CACHE_SUCCEEDED && state != HTMLElementCacheState.CACHE_REQUESTED; - }); + // return HtmlAssistant.isCacheStateEqual(state, [ + // HTMLElementCacheState.ORIGINAL, + // HTMLElementCacheState.ORIGINAL_SRC_REMOVED, + // HTMLElementCacheState.CACHE_FAILED + // ]); - if (imageElements.length == 0) - Log(`\tAborting: All relevant images are either already using cache or are waiting for it.`) - - return imageElements; + // Difference between this and the above is that ORIGINAL matches all unprocessed elements. + return !HtmlAssistant.isCacheStateEqual(state, [ + HTMLElementCacheState.REQUESTING, + HTMLElementCacheState.REQUESTING_DOWNLOADING, + HTMLElementCacheState.CACHE_SUCCEEDED, + HTMLElementCacheState.INVALID + ]); } private editorViewUpdateListener(update: ViewUpdate) { - Log(`editorViewUpdateListener`) - - const proceed = this.proceed(); - if (!proceed) - return; - Log(`\t${this.viewModeString(proceed.markdownView, update.view)}`); + const processingPass = ProcessingPass.beginFromViewUpdate(this.app, update, () => { + // There is no file to work with. All that can be done is to cancel loading all external urls. + const imageElements = HtmlAssistant.findRelevantImagesToProcess(update.view.contentDOM, true, (imageElement) => { + const src = imageElement.getAttribute(HTMLElementAttribute.SRC); + return src !== null && Url.isValid(src) && Url.isExternal(src); + }); + HtmlAssistant.cancelImageLoading(imageElements); + }); - const imageElements = this.imageElements(update.view.contentDOM); - if (imageElements.length == 0) - return; - - if (ENV.processOnAllViewUpdateChanges || (update.docChanged || update.viewportChanged || update.geometryChanged)) { - if (!ENV.processOnAllViewUpdateChanges) - Log(`\tdocChanged: ${update.docChanged}\n\tviewportChanged: ${update.viewportChanged}\n\tselectionSet: ${update.selectionSet}\n\tfocusChanged: ${update.focusChanged}\n\tgeometryChanged: ${update.geometryChanged}\n\theightChanged: ${update.heightChanged}`); + if (processingPass) { - this.handleImages(HtmlAssistant.preparedImageElements(imageElements), proceed.associatedFile); - } - else { - if (!ENV.processOnAllViewUpdateChanges) - Log(`\tskipped \`handleImages\`: \n\tviewportChanged: ${update.viewportChanged}\n\tselectionSet: ${update.selectionSet}\n\tfocusChanged: ${update.focusChanged}\n\tgeometryChanged: ${update.geometryChanged}\n\theightChanged: ${update.heightChanged}`); + // Elements in DOM at this stage might be in states in which the `src` attribute has been removed. Therefore the `src` attribute is not required when finding image elements. + const imageElements = HtmlAssistant.findRelevantImagesToProcess(update.view.contentDOM, false, (imageElement) => { + const src = imageElement.getAttribute(HTMLElementAttribute.SRC); + + // 1. The user is editing the link (causing the source attribute to change) which resets the element's state. + // External urls are only accepted in the element's original state. + if (update.docChanged && src && Url.isExternal(src) && HtmlAssistant.cacheState(imageElement) != HTMLElementCacheState.ORIGINAL) + HtmlAssistant.resetElement(imageElement); // Set state to "untouched". + + // 2. As all images are retained in each pass, even though elements that are already cached are excluded from further processing, they still need to be retained. + if (HtmlAssistant.cacheState(imageElement) == HTMLElementCacheState.CACHE_SUCCEEDED) { + const src = HtmlAssistant.originalSrc(imageElement); + console.assert(src !== null, "Expected original source dataset"); + if (src) + processingPass.retainCacheFromRequest({ source: src, requesterPath: processingPass.associatedFile.path }); + } + + // 3. Start filtering: If the image element has a src, only keep external urls; if not, keep all. + let isSrcOk = src !== null ? Url.isValid(src) && Url.isExternal(src) : true; + + // 4. If the url was ok, then remove all states that have passed this stage already. + return isSrcOk && this.filterIrrelevantCacheStates(imageElement); + }); + + if (imageElements.length > 0) { + HtmlAssistant.cancelImageLoading(imageElements); + this.requestCache(imageElements, processingPass); + } + else { + // Special case when the user deletes an existing embed and all other image elements were filtered out: there are either no other embeds or all the other are already done. + if (update.docChanged) + processingPass.end(this.cacheManager); + else + processingPass.abort(); + } } } /** + * - Will not be called if never in Read mode since there's no need to render Markdown. + * - In Read mode, it will always be called after the update listener, {@link editorViewUpdateListener}, as it might make changes. * - * @param element A chunk of html. + * @param element A chunk of HTML. * @param context */ - processReadingMode(element: HTMLElement, context: MarkdownPostProcessorContext) { - Log(`processReadingMode`) - - const proceed = this.proceed(context); - if (!proceed) - return; - - Log(`\t${this.viewModeString(proceed.markdownView)}`); - - const imageElements = this.imageElements(element); - if (imageElements.length == 0) - return; + private postProcessReadingModeHtml(element: HTMLElement, context: MarkdownPostProcessorContext) { - //console.log(element); - //console.log(context.getSectionInfo(element)?.text); // This is the Markdown text + const processingPass = ProcessingPass.beginFromPostProcessorContext(this.app, context); + const imageElements = HtmlAssistant.findRelevantImagesToProcess(element, true, (imageElement) => { + const src = imageElement.getAttribute(HTMLElementAttribute.SRC); + return src !== null && Url.isValid(src) && Url.isExternal(src); + }); - this.handleImages(HtmlAssistant.preparedImageElements(imageElements), proceed.associatedFile); + if (imageElements.length > 0) { + HtmlAssistant.cancelImageLoading(imageElements); + this.requestCache(imageElements, processingPass); + } + else { + processingPass.abort(); + } } /** - * Initiates a call to the {@link CacheManager} for each image element and changing the src to the cached file. * - * @param imageElements All images in the post proceessed HTML. - * @param filePath Path to the file containing this HTML. + * @param imageElements + * @param processingPass + * @returns */ - async handleImages(imageElements: HTMLImageElement[], associatedFile: TFile) { + async requestCache(imageElements: HTMLImageElement[], processingPass: ProcessingPass) { - imageElements = imageElements.length == 0 - ? [] - : imageElements.filter((imageElement) => HtmlAssistant.cacheState(imageElement) == HTMLElementCacheState.ORIGINAL_CANCELLED); + imageElements = imageElements.filter((imageElement) => HtmlAssistant.isElementCacheStateEqual(imageElement, [HTMLElementCacheState.ORIGINAL_SRC_REMOVED, HTMLElementCacheState.CACHE_FAILED])); - Log(`handleImages\n\tGot ${imageElements.length} elements to populate.`); + Log(`requestCache\n\tGot ${imageElements.length} elements to populate.`); if (imageElements.length == 0) return; - //#region Prepare requests and group equal elements (making it one request per file, not per element). - - const groupedByRequest: Record, - done: boolean, - }> = {}; - - imageElements.forEach((imageElement) => { - HtmlAssistant.setCacheState(imageElement, HTMLElementCacheState.CACHE_REQUESTED); - - const src = HtmlAssistant.imageElementOriginalSrc(imageElement); - - if (src) { - const key = CacheManager.cacheKeyFromOriginalSrc(src); - const group = groupedByRequest[key]; - const alt = imageElement.hasAttribute(HTMLElementAttribute.ALT) ? imageElement.alt : ""; - - if (group) { - group.data.imageElements.push(imageElement); - group.data.originalAlts.push(alt); - } - else { - const request = { key: src }; - const validationError = this.cacheManager.validateRequest(request); - if (validationError) { - console.error(`Image element does not have a source. Omitting.`) - HtmlAssistant.setFailed(imageElement); - } - else { - groupedByRequest[key] = { - request, - filePath: associatedFile.path, - data: { imageElements: [imageElement], originalAlts: [alt] }, - done: false, - }; - } - } - } - else { - console.error(`Image element does not have a source. Omitting.`) - HtmlAssistant.setFailed(imageElement); - } - - }); - //#endregion - - const requestGroups = Object.values(groupedByRequest); // TODO: Can be array. - let numberFilesToDownload = 0; + const requestGroups = groupRequests(imageElements, this.cacheManager, processingPass); + // First try to get src from local cache. for (const requestGroup of requestGroups) { - const cacheItem = await this.cacheManager.existingCachedItem(requestGroup.request); + const existingCacheResult = await this.cacheManager.existingCache(requestGroup.request, true); + Log(`requestCache: ${existingCacheResult.item ? "Found" : "Did not find"} key ${existingCacheResult.cacheKey}. ${existingCacheResult.fileExists === undefined ? "Unknown if file exists." : `${existingCacheResult.fileExists ? "File exists." : "File does not exist."}`}`); + if (existingCacheResult.item) + await handleRequestGroup(existingCacheResult.item, requestGroup); + }; - if (cacheItem) { - requestGroup.data.imageElements.forEach((imageElement: HTMLImageElement) => - HtmlAssistant.setSuccess(imageElement, cacheItem.filePath)); - requestGroup.done = true; - Log(`handleImages:\n\t Used available cache: ${GetCacheKey(cacheItem.metadata.hash)}`) - } - else { - numberFilesToDownload++; - } + const remainingRequestGroups = requestGroups.filter((requestGroup) => !requestGroup.cacheFileFound); + let numberOfRemainingDownloads = remainingRequestGroups.length; + + if (numberOfRemainingDownloads == 0) { + Log(`requestCache: All images were in cache. Done ${processingPass.passID}`); + processingPass.end(this.cacheManager); + return; } + // If we are here, some images need to be downloaded. + + // Show download notice if (this.settingsManager.settings.noticeOnDownload) { - if (numberFilesToDownload > 0) { - const notice = `↓ ${numberFilesToDownload}`;// file${numberFilesToDownload != 1 ? `s` : ``}`; + processingPass.runInUpdatePass(() => { + const notice = `↓ ${numberOfRemainingDownloads}`; new Notice(notice, undefined, this.settingsManager.settings.omitNameInNotice); - Log(`${associatedFile.path}: ${notice}`); - } + }); } - - requestGroups - .filter((requestGroup) => !requestGroup.done) - .forEach((requestGroup) => { - const imageElements: HTMLImageElement[] = requestGroup.data.imageElements; - imageElements.forEach((ie) => { - HtmlAssistant.setIcon(ie, HtmlAssistant.ENCODED_LOADING_ICON); - ie.setAttribute(HTMLElementAttribute.ALT, "Loading..."); + remainingRequestGroups.forEach((requestGroup) => { + + for (const imageElement of requestGroup.imageElements) { + HtmlAssistant.setCacheState(imageElement, HTMLElementCacheState.REQUESTING_DOWNLOADING); + HtmlAssistant.setLoadingIcon(imageElement); + imageElement.setAttribute(HTMLElementAttribute.ALT, "Loading..."); + }; + + // Pass `true` flag because it is known at this point that the cache doesn't exist. + this.cacheManager.getCache(requestGroup.request, true, async (result) => { + + numberOfRemainingDownloads--; + const imageElements: HTMLImageElement[] = requestGroup.imageElements; + + // Restore alt texts. + imageElements.forEach((imageElement, index) => { + const originalAlt = requestGroup.altAttributeValues[index]; + if (originalAlt.length > 0) + imageElement.setAttribute(HTMLElementAttribute.ALT, originalAlt); + else + imageElement.removeAttribute(HTMLElementAttribute.ALT); // TODO: Chromium sets it to the original url if removed? }); - // Fire and forget - this.cacheManager.getCache(requestGroup.request, (result) => { + if (result.item) { + Log(`requestCache:\n\tSettings src on ${imageElements.length} images\n\t${result.item.metadata.f.n}\n\t${processingPass.isInPostProcessingPass ? `In HTML post processor` : `In edit listener`}`); + if (result.fileExists === true) + await handleRequestGroup(result.item, requestGroup); + else + imageElements.forEach((imageElement) => HtmlAssistant.setFailed(imageElement)); + } + else { + imageElements.forEach((imageElement) => HtmlAssistant.setInvalid(imageElement)); - const imageElements: HTMLImageElement[] = requestGroup.data.imageElements; + if (ENV.debugLog && (result.error instanceof CacheFetchError || result.error instanceof CacheTypeError)) + console.error(`requestCache:\n\t${result.error.name}`, result.error); - // Restore alt texts. - const originalAlts: string[] = requestGroup.data.originalAlts; - imageElements.forEach((imageElement, index) => { - const originalAlt = originalAlts[index]; - if (originalAlt.length > 0) - imageElement.setAttribute(HTMLElementAttribute.ALT, originalAlt); - else - imageElement.removeAttribute(HTMLElementAttribute.ALT); // TODO: Chromium sets it to the original url if removed? - }); + if (!(result.error instanceof CacheFetchError || result.error instanceof CacheTypeError)) + console.error("requestCache:\n\tFailed to fetch cache", result.error); + } - if (result.item) { - Log(`handleImages:\n\tCache result received. Settings src on ${imageElements.length} images\n\t${result.item.metadata.file.name}`) - const filePath = result.item.filePath; - imageElements.forEach((imageElement) => HtmlAssistant.setSuccess(imageElement, filePath)); + if (numberOfRemainingDownloads == 0) { + processingPass.end(this.cacheManager); + if (result.error instanceof CacheFetchError && result.error.isInternetDisconnected) + new Notice("No internet connection."); + } + }); + }); + + /** + * - Uses the cache item to load each image element in the request group. + * - Sets the resulting {@link HTMLElementCacheState}. + * - Sets {@link RequestGroup.cacheFileFound} + * - Marks the src reference as retained. + * + * @param cacheItem + * @param requestGroup + */ + async function handleRequestGroup(cacheItem: CacheItem, requestGroup: RequestGroup) { + const imageElements = requestGroup.imageElements; + const errorResult = await HtmlAssistant.loadImages(imageElements, cacheItem.resourcePath); + + if (errorResult) { + // File as not found on disk. + console.error(errorResult.error); + //imageElements.forEach((imageElement) => HtmlAssistant.setFailed(imageElement)); + } + else { + imageElements.forEach((imageElement) => HtmlAssistant.setCacheState(imageElement, HTMLElementCacheState.CACHE_SUCCEEDED)); + } + + requestGroup.cacheFileFound = errorResult ? false : true; + + if (requestGroup.cacheFileFound) { + Log(`requestCache:handleRequestGroup: Found cached image: ${CacheManager.createCacheKeyFromMetadata(cacheItem.metadata)}, ID ${processingPass.passID} 📦📦📦`); + processingPass.retainCacheFromRequest(requestGroup.request); + } + } + + /** + * - Create {@link CacheRequest} for each unique image source. + * - Group images per unique request. + * - Set the state to {@link HTMLElementCacheState.REQUESTING} on each element. + * - Retain info that could be overwritten while making the request to be able to restore it afterwards. + * + * @param imageElements + * @param processingPass + * @returns + */ + function groupRequests(imageElements: HTMLImageElement[], cacheManager: CacheManager, processingPass: ProcessingPass) { + + const groupedByRequest: Record = {}; + + imageElements.forEach((imageElement) => { + HtmlAssistant.setCacheState(imageElement, HTMLElementCacheState.REQUESTING); + + const src = HtmlAssistant.imageElementOriginalSrc(imageElement); + + if (src) { + const key = CacheManager.createCacheKeyFromOriginalSrc(src); + const group = groupedByRequest[key]; + const alt = imageElement.hasAttribute(HTMLElementAttribute.ALT) ? imageElement.alt : ""; + + if (group) { + group.imageElements.push(imageElement); + group.altAttributeValues.push(alt); } else { - console.error("Failed to fetch cache", result.error); - imageElements.forEach((imageElement) => HtmlAssistant.setFailed(imageElement)); + const request: CacheRequest = { source: src, requesterPath: processingPass.associatedFile.path }; + const validationError = cacheManager.validateRequest(request); + if (validationError) { + console.error(`Cache request failed validation.`, validationError) + HtmlAssistant.setInvalid(imageElement); + } + else { + groupedByRequest[key] = { + request, + imageElements: [imageElement], + altAttributeValues: [alt], + cacheFileFound: false, + }; + } } - }, true); + } + else { + console.error(`Image element does not have a source. Omitting.`) + HtmlAssistant.setInvalid(imageElement); + } }); + + return Object.values(groupedByRequest); + } } //#endregion } +/** + * Group of images that share the same cash request. + */ +interface RequestGroup { + request: CacheRequest, + imageElements: HTMLImageElement[], + altAttributeValues: string[], + /** + * Set to true to indicate that the cache was available and there's no need to download it. + * @description This is just a helper because one could iterate the image elements and check the {@link HTMLElementCacheState} to find out. + */ + cacheFileFound: boolean, +} + + diff --git a/tsconfig.json b/tsconfig.json index c44b729..0eadc22 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,22 +1,17 @@ { "compilerOptions": { - "baseUrl": ".", + "baseUrl": "src", "inlineSourceMap": true, "inlineSources": true, "module": "ESNext", - "target": "ES6", + "target": "ES2022", "allowJs": true, "noImplicitAny": true, "moduleResolution": "node", "importHelpers": true, "isolatedModules": true, "strictNullChecks": true, - "lib": [ - "DOM", - "ES5", - "ES6", - "ES7" - ] + "lib": ["DOM", "ES2022"] }, "include": [ "**/*.ts"