Enhance first sync from remote to make less requests

package.json

@@ -34,6 +34,7 @@
     "@codemirror/state": "^6.5.2",
     "@codemirror/view": "^6.36.2",
     "@uiw/react-codemirror": "^4.23.8",
+    "@zip.js/zip.js": "^2.7.60",
     "codemirror": "^6.0.1",
     "file-type": "^20.4.1",
     "react": "^19.0.0",

pnpm-lock.yaml

@@ -20,6 +20,9 @@ importers:
       '@uiw/react-codemirror':
         specifier: ^4.23.8
         version: 4.23.8(@babel/runtime@7.26.9)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.10.8)(@codemirror/lint@6.8.4)(@codemirror/search@6.5.9)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.2)(codemirror@6.0.1)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@zip.js/zip.js':
+        specifier: ^2.7.60
+        version: 2.7.60
       codemirror:
         specifier: ^6.0.1
         version: 6.0.1
@@ -645,6 +648,10 @@ packages:
       react: '>=16.8.0'
       react-dom: '>=16.8.0'
 
+  '@zip.js/zip.js@2.7.60':
+    resolution: {integrity: sha512-vA3rLyqdxBrVo1FWSsbyoecaqWTV+vgPRf0QKeM7kVDG0r+lHUqd7zQDv1TO9k4BcAoNzNDSNrrel24Mk6addA==}
+    engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=16.5.0'}
+
   acorn-jsx@5.3.2:
     resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==}
     peerDependencies:
@@ -1710,6 +1717,8 @@ snapshots:
       - '@codemirror/lint'
       - '@codemirror/search'
 
+  '@zip.js/zip.js@2.7.60': {}
+
   acorn-jsx@5.3.2(acorn@8.14.0):
     dependencies:
       acorn: 8.14.0

src/github/client.ts

@@ -254,4 +254,27 @@ export default class GithubClient {
       );
     }
   }
+
+  /**
+   * Downloads the repository as a ZIP archive from GitHub.
+   *
+   * @returns The archive contents as an ArrayBuffer
+   */
+  async downloadRepositoryArchive(): Promise<ArrayBuffer> {
+    const res = await requestUrl({
+      url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/zipball/${this.settings.githubBranch}`,
+      headers: this.headers(),
+      method: "GET",
+      throw: false,
+    });
+
+    if (res.status < 200 || res.status >= 400) {
+      await this.logger.error("Failed to download zip archive", res);
+      throw new GithubAPIError(
+        res.status,
+        `Failed to download zip archive, status ${res.status}`,
+      );
+    }
+    return res.arrayBuffer;
+  }
 }

src/sync-manager.ts

@@ -21,6 +21,7 @@ import Logger from "./logger";
 import { decodeBase64String } from "./utils";
 import GitHubSyncPlugin from "./main";
 import { fileTypeFromBuffer } from "file-type";
+import { BlobReader, Entry, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js";
 
 interface SyncAction {
   type: "upload" | "download" | "delete_local" | "delete_remote";
@@ -187,24 +188,42 @@ export default class SyncManager {
     treeSha: string,
   ) {
     await this.logger.info("Starting first sync from remote files");
+
+    // We want to avoid getting throttled by GitHub, so instead of making a request for each
+    // file we download the whole repository as a ZIP file and extract it in the vault.
+    // We exclude config dir files if the user doesn't want to sync those.
+    const zipBuffer = await this.client.downloadRepositoryArchive();
+    const zipBlob = new Blob([zipBuffer]);
+    const reader = new ZipReader(new BlobReader(zipBlob));
+    const entries = await reader.getEntries();
+
     await Promise.all(
-      Object.keys(files)
-        .filter((filePath: string) => {
-          if (
-            this.settings.syncConfigDir &&
-            filePath.startsWith(this.vault.configDir) &&
-            filePath !== `${this.vault.configDir}/${MANIFEST_FILE_NAME}`
-          ) {
-            // Include files in the config dir only if the user has enabled it.
-            // The metadata file must always be synced.
-            return false;
-          }
-          return true;
-        })
-        .map(async (filePath: string) => {
-          await this.downloadFile(files[filePath], Date.now());
-        }),
+      entries.map(async (entry: Entry) => {
+        if (
+          this.settings.syncConfigDir &&
+          entry.filename.startsWith(this.vault.configDir) &&
+          entry.filename !== `${this.vault.configDir}/${MANIFEST_FILE_NAME}`
+        ) {
+          // If the user doesn't want to sync the config directory ignore it completely,
+          // apart from the metadata file as that must always be synced so the plugin behaves
+          // correctly.
+          return;
+        }
+
+        const writer = new Uint8ArrayWriter();
+        await entry.getData!(writer);
+        const data = await writer.getData();
+        const dir = entry.filename.split("/").splice(0, -1).join("/");
+        if (dir !== "") {
+          await this.vault.adapter.mkdir(normalizePath(dir));
+        }
+        await this.vault.adapter.writeBinary(
+          normalizePath(entry.filename),
+          data,
+        );
+      }),
     );
+
     const newTreeFiles = Object.keys(files)
       .map((filePath: string) => ({
         path: files[filePath].path,