From 6c9b8625c33730d56ad5e8452704fc53ed400249 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Wed, 16 Apr 2025 14:22:39 +0200 Subject: [PATCH] Enhance first sync from remote to make less requests --- package.json | 1 + pnpm-lock.yaml | 9 ++++++++ src/github/client.ts | 23 ++++++++++++++++++++ src/sync-manager.ts | 51 ++++++++++++++++++++++++++++++-------------- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index ca53c31..e9091ba 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@codemirror/state": "^6.5.2", "@codemirror/view": "^6.36.2", "@uiw/react-codemirror": "^4.23.8", + "@zip.js/zip.js": "^2.7.60", "codemirror": "^6.0.1", "file-type": "^20.4.1", "react": "^19.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd49143..aeacfcf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@uiw/react-codemirror': specifier: ^4.23.8 version: 4.23.8(@babel/runtime@7.26.9)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.10.8)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.9)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.2)(codemirror@6.0.1)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + '@zip.js/zip.js': + specifier: ^2.7.60 + version: 2.7.60 codemirror: specifier: ^6.0.1 version: 6.0.1 @@ -645,6 +648,10 @@ packages: react: '>=16.8.0' react-dom: '>=16.8.0' + '@zip.js/zip.js@2.7.60': + resolution: {integrity: sha512-vA3rLyqdxBrVo1FWSsbyoecaqWTV+vgPRf0QKeM7kVDG0r+lHUqd7zQDv1TO9k4BcAoNzNDSNrrel24Mk6addA==} + engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=16.5.0'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1710,6 +1717,8 @@ snapshots: - '@codemirror/lint' - '@codemirror/search' + '@zip.js/zip.js@2.7.60': {} + acorn-jsx@5.3.2(acorn@8.14.0): dependencies: acorn: 8.14.0 diff --git a/src/github/client.ts b/src/github/client.ts index 9b24267..4181547 100644 --- a/src/github/client.ts +++ b/src/github/client.ts @@ -254,4 +254,27 @@ export default class GithubClient { ); } } + + /** + * Downloads the repository as a ZIP archive from GitHub. + * + * @returns The archive contents as an ArrayBuffer + */ + async downloadRepositoryArchive(): Promise { + const res = await requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/zipball/${this.settings.githubBranch}`, + headers: this.headers(), + method: "GET", + throw: false, + }); + + if (res.status < 200 || res.status >= 400) { + await this.logger.error("Failed to download zip archive", res); + throw new GithubAPIError( + res.status, + `Failed to download zip archive, status ${res.status}`, + ); + } + return res.arrayBuffer; + } } diff --git a/src/sync-manager.ts b/src/sync-manager.ts index e7eb4a0..5869ee9 100644 --- a/src/sync-manager.ts +++ b/src/sync-manager.ts @@ -21,6 +21,7 @@ import Logger from "./logger"; import { decodeBase64String } from "./utils"; import GitHubSyncPlugin from "./main"; import { fileTypeFromBuffer } from "file-type"; +import { BlobReader, Entry, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js"; interface SyncAction { type: "upload" | "download" | "delete_local" | "delete_remote"; @@ -187,24 +188,42 @@ export default class SyncManager { treeSha: string, ) { await this.logger.info("Starting first sync from remote files"); + + // We want to avoid getting throttled by GitHub, so instead of making a request for each + // file we download the whole repository as a ZIP file and extract it in the vault. + // We exclude config dir files if the user doesn't want to sync those. + const zipBuffer = await this.client.downloadRepositoryArchive(); + const zipBlob = new Blob([zipBuffer]); + const reader = new ZipReader(new BlobReader(zipBlob)); + const entries = await reader.getEntries(); + await Promise.all( - Object.keys(files) - .filter((filePath: string) => { - if ( - this.settings.syncConfigDir && - filePath.startsWith(this.vault.configDir) && - filePath !== `${this.vault.configDir}/${MANIFEST_FILE_NAME}` - ) { - // Include files in the config dir only if the user has enabled it. - // The metadata file must always be synced. - return false; - } - return true; - }) - .map(async (filePath: string) => { - await this.downloadFile(files[filePath], Date.now()); - }), + entries.map(async (entry: Entry) => { + if ( + this.settings.syncConfigDir && + entry.filename.startsWith(this.vault.configDir) && + entry.filename !== `${this.vault.configDir}/${MANIFEST_FILE_NAME}` + ) { + // If the user doesn't want to sync the config directory ignore it completely, + // apart from the metadata file as that must always be synced so the plugin behaves + // correctly. + return; + } + + const writer = new Uint8ArrayWriter(); + await entry.getData!(writer); + const data = await writer.getData(); + const dir = entry.filename.split("/").splice(0, -1).join("/"); + if (dir !== "") { + await this.vault.adapter.mkdir(normalizePath(dir)); + } + await this.vault.adapter.writeBinary( + normalizePath(entry.filename), + data, + ); + }), ); + const newTreeFiles = Object.keys(files) .map((filePath: string) => ({ path: files[filePath].path,