From 13383dc65ce470092f93eb1ba3db802659fea935 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Thu, 17 Apr 2025 12:16:47 +0200 Subject: [PATCH] Change all GithubClient method to support retry and slightly change arguments type --- src/github/client.ts | 421 ++++++++++++++++++++++++++++++------------- src/sync-manager.ts | 39 ++-- 2 files changed, 315 insertions(+), 145 deletions(-) diff --git a/src/github/client.ts b/src/github/client.ts index 4181547..b09fa4b 100644 --- a/src/github/client.ts +++ b/src/github/client.ts @@ -1,6 +1,7 @@ import { requestUrl } from "obsidian"; import Logger from "src/logger"; import { GitHubSyncSettings } from "src/settings/settings"; +import { retryUntil } from "src/utils"; export type RepoContent = { files: { [key: string]: GetTreeResponseItem }; @@ -75,22 +76,35 @@ export default class GithubClient { /** * Gets the content of the repo. * + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) * @returns Array of files in the directory in the remote repo */ - async getRepoContent(): Promise { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/trees/${this.settings.githubBranch}?recursive=1`, - headers: this.headers(), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to get repo content", res); + async getRepoContent({ + retry = false, + maxRetries = 5, + } = {}): Promise { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/trees/${this.settings.githubBranch}?recursive=1`, + headers: this.headers(), + throw: false, + }); + }, + (res) => res.status !== 422, // Retry condition: only retry on 422 status + retry ? maxRetries : 0, // Use 0 retries if retry is false + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to get repo content", response); throw new GithubAPIError( - res.status, - `Failed to get repo content, status ${res.status}`, + response.status, + `Failed to get repo content, status ${response.status}`, ); } - const files = res.json.tree + + const files = response.json.tree .filter((file: GetTreeResponseItem) => file.type === "blob") .reduce( ( @@ -99,80 +113,168 @@ export default class GithubClient { ) => ({ ...acc, [file.path]: file }), {}, ); - return { files, sha: res.json.sha }; + return { files, sha: response.json.sha }; } - async createTree(tree: { tree: NewTreeRequestItem[]; base_tree: string }) { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/trees`, - headers: this.headers(), - method: "POST", - body: JSON.stringify(tree), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to create tree", res); + /** + * Creates a new tree in the GitHub repository. + * + * @param tree The tree object to create + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) + * @returns The SHA of the created tree + */ + async createTree({ + tree, + retry = false, + maxRetries = 5, + }: { + tree: { tree: NewTreeRequestItem[]; base_tree: string }; + retry?: boolean; + maxRetries?: number; + }) { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/trees`, + headers: this.headers(), + method: "POST", + body: JSON.stringify(tree), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to create tree", response); throw new GithubAPIError( - res.status, - `Failed to create tree, status ${res.status}`, + response.status, + `Failed to create tree, status ${response.status}`, ); } - return res.json.sha; + return response.json.sha; } - async createCommit(message: string, treeSha: string, parent: string) { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/commits`, - headers: this.headers(), - method: "POST", - body: JSON.stringify({ - message: message, - tree: treeSha, - parents: [parent], - }), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to create commit", res); + /** + * Creates a new commit in the repository. + * + * @param message The commit message + * @param treeSha The SHA of the tree + * @param parent The SHA of the parent commit + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) + * @returns The SHA of the created commit + */ + async createCommit({ + message, + treeSha, + parent, + retry = false, + maxRetries = 5, + }: { + message: string; + treeSha: string; + parent: string; + retry?: boolean; + maxRetries?: number; + }): Promise { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/commits`, + headers: this.headers(), + method: "POST", + body: JSON.stringify({ + message: message, + tree: treeSha, + parents: [parent], + }), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to create commit", response); throw new GithubAPIError( - res.status, - `Failed to create commit, status ${res.status}`, + response.status, + `Failed to create commit, status ${response.status}`, ); } - return res.json.sha; + return response.json.sha; } - async getBranchHeadSha() { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/refs/heads/${this.settings.githubBranch}`, - headers: this.headers(), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to get branch head sha", res); + /** + * Gets the SHA of the branch head. + * + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) + * @returns The SHA of the branch head + */ + async getBranchHeadSha({ retry = false, maxRetries = 5 } = {}) { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/refs/heads/${this.settings.githubBranch}`, + headers: this.headers(), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to get branch head sha", response); throw new GithubAPIError( - res.status, - `Failed to get branch head sha, status ${res.status}`, + response.status, + `Failed to get branch head sha, status ${response.status}`, ); } - return res.json.object.sha; + return response.json.object.sha; } - async updateBranchHead(sha: string) { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/refs/heads/${this.settings.githubBranch}`, - headers: this.headers(), - method: "PATCH", - body: JSON.stringify({ - sha: sha, - }), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to update branch head sha", res); + /** + * Updates the branch head to point to a new commit. + * + * @param sha The SHA of the commit to point to + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) + */ + async updateBranchHead({ + sha, + retry = false, + maxRetries = 5, + }: { + sha: string; + retry?: boolean; + maxRetries?: number; + }) { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/refs/heads/${this.settings.githubBranch}`, + headers: this.headers(), + method: "PATCH", + body: JSON.stringify({ + sha: sha, + }), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to update branch head sha", response); throw new GithubAPIError( - res.status, - `Failed to update branch head sha, status ${res.status}`, + response.status, + `Failed to update branch head sha, status ${response.status}`, ); } } @@ -182,75 +284,131 @@ export default class GithubClient { * * @param content The content of the blob to upload * @param encoding Content encoding, can be "utf-8" or "base64". Defaults to "base64" + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) * @returns The SHA of the newly uploaded blob */ - async createBlob( - content: string, - encoding: "utf-8" | "base64" = "base64", - ): Promise { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs`, - headers: this.headers(), - method: "POST", - body: JSON.stringify({ content, encoding }), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to create blob", res); + async createBlob({ + content, + encoding = "base64", + retry = false, + maxRetries = 5, + }: { + content: string; + encoding?: "utf-8" | "base64"; + retry?: boolean; + maxRetries?: number; + }): Promise { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs`, + headers: this.headers(), + method: "POST", + body: JSON.stringify({ content, encoding }), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to create blob", response); throw new GithubAPIError( - res.status, - `Failed to create blob, status ${res.status}`, + response.status, + `Failed to create blob, status ${response.status}`, ); } return { - sha: res.json["sha"], + sha: response.json["sha"], }; } /** * Gets a blob from its sha - * @param url blob sha + * + * @param sha The SHA of the blob + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) + * @returns The blob file */ - async getBlob(sha: string): Promise { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs/${sha}`, - headers: this.headers(), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to get blob", res); + async getBlob({ + sha, + retry = false, + maxRetries = 5, + }: { + sha: string; + retry?: boolean; + maxRetries?: number; + }): Promise { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs/${sha}`, + headers: this.headers(), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to get blob", response); throw new GithubAPIError( - res.status, - `Failed to get blob, status ${res.status}`, + response.status, + `Failed to get blob, status ${response.status}`, ); } - return res.json; + return response.json; } /** * Create a new file in the repo, the content must be base64 encoded or the request will fail. * - * @param message commit message - * @param path path to create in the repo - * @param content base64 encoded content of the file + * @param path Path to create in the repo + * @param content Base64 encoded content of the file + * @param message Commit message + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) */ - async createFile(path: string, content: string, message: string) { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/contents/${path}`, - headers: this.headers(), - method: "PUT", - body: JSON.stringify({ - message: message, - content: content, - branch: this.settings.githubBranch, - }), - throw: false, - }); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to create file", res); + async createFile({ + path, + content, + message, + retry = false, + maxRetries = 5, + }: { + path: string; + content: string; + message: string; + retry?: boolean; + maxRetries?: number; + }) { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/contents/${path}`, + headers: this.headers(), + method: "PUT", + body: JSON.stringify({ + message: message, + content: content, + branch: this.settings.githubBranch, + }), + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); + + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to create file", response); throw new GithubAPIError( - res.status, - `Failed to create file, status ${res.status}`, + response.status, + `Failed to create file, status ${response.status}`, ); } } @@ -258,23 +416,34 @@ export default class GithubClient { /** * Downloads the repository as a ZIP archive from GitHub. * + * @param retry Whether to retry the request on failure (default: false) + * @param maxRetries Maximum number of retry attempts (default: 5) * @returns The archive contents as an ArrayBuffer */ - async downloadRepositoryArchive(): Promise { - const res = await requestUrl({ - url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/zipball/${this.settings.githubBranch}`, - headers: this.headers(), - method: "GET", - throw: false, - }); + async downloadRepositoryArchive({ + retry = false, + maxRetries = 5, + } = {}): Promise { + const response = await retryUntil( + async () => { + return requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/zipball/${this.settings.githubBranch}`, + headers: this.headers(), + method: "GET", + throw: false, + }); + }, + (res) => res.status !== 422, + retry ? maxRetries : 0, + ); - if (res.status < 200 || res.status >= 400) { - await this.logger.error("Failed to download zip archive", res); + if (response.status < 200 || response.status >= 400) { + await this.logger.error("Failed to download zip archive", response); throw new GithubAPIError( - res.status, - `Failed to download zip archive, status ${res.status}`, + response.status, + `Failed to download zip archive, status ${response.status}`, ); } - return res.arrayBuffer; + return response.arrayBuffer; } } diff --git a/src/sync-manager.ts b/src/sync-manager.ts index 62c8514..e616bbd 100644 --- a/src/sync-manager.ts +++ b/src/sync-manager.ts @@ -142,11 +142,11 @@ export default class SyncManager { // API doesn't let us create a new tree when the repo is empty. // So we create a the manifest file as the first commit, since we're going // to create that in any case right after this. - await this.client.createFile( - `${this.vault.configDir}/${MANIFEST_FILE_NAME}`, - "", - "First sync", - ); + await this.client.createFile({ + path: `${this.vault.configDir}/${MANIFEST_FILE_NAME}`, + content: "", + message: "First sync", + }); // Now get the repo content again cause we know for sure it will return a // valid sha that we can use to create the first sync commit. res = await this.client.getRepoContent(); @@ -341,7 +341,7 @@ export default class SyncManager { throw new Error("Remote manifest is missing"); } - const blob = await this.client.getBlob(manifest.sha); + const blob = await this.client.getBlob({ sha: manifest.sha }); const remoteMetadata: Metadata = JSON.parse( decodeBase64String(blob.content), ); @@ -537,9 +537,9 @@ export default class SyncManager { // Load contents in parallel const [remoteContent, localContent] = await Promise.all([ await (async () => { - const res = await this.client.getBlob( - filesMetadata[filePath].sha!, - ); + const res = await this.client.getBlob({ + sha: filesMetadata[filePath].sha!, + }); return decodeBase64String(res.content); })(), await this.vault.adapter.read(normalizePath(filePath)), @@ -760,8 +760,9 @@ export default class SyncManager { // we first need to create a Git blob by uploading the file, then // we must update the tree item to point the SHA to the blob we just created. const buffer = await this.vault.adapter.readBinary(filePath); - const hash = arrayBufferToBase64(buffer); - const { sha } = await this.client.createBlob(hash); + const { sha } = await this.client.createBlob({ + content: arrayBufferToBase64(buffer), + }); treeFiles[filePath].sha = sha; // Can't have both sha and content set, so we delete it delete treeFiles[filePath].content; @@ -781,18 +782,18 @@ export default class SyncManager { ), base_tree: baseTreeSha, }; - const newTreeSha = await this.client.createTree(newTree); + const newTreeSha = await this.client.createTree({ tree: newTree }); const branchHeadSha = await this.client.getBranchHeadSha(); - const commitSha = await this.client.createCommit( + const commitSha = await this.client.createCommit({ // TODO: Make this configurable or find a nicer commit message - "Sync", - newTreeSha, - branchHeadSha, - ); + message: "Sync", + treeSha: newTreeSha, + parent: branchHeadSha, + }); - await this.client.updateBranchHead(commitSha); + await this.client.updateBranchHead({ sha: commitSha }); // Update the local content of all files that had conflicts we resolved await Promise.all( @@ -819,7 +820,7 @@ export default class SyncManager { // File already exists and has the same SHA, no need to download it again. return; } - const blob = await this.client.getBlob(file.sha); + const blob = await this.client.getBlob({ sha: file.sha }); const normalizedPath = normalizePath(file.path); const fileFolder = normalizePath( normalizedPath.split("/").slice(0, -1).join("/"),