From 5f0c8b1ed24a9e8d4dd12b7d8273233ce7196bb9 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Fri, 21 Mar 2025 17:03:13 +0100 Subject: [PATCH] Fix binary files failing to upload to GitHub --- package.json | 1 + pnpm-lock.yaml | 73 ++++++++++++++++++++++++++++++++++++++++++++ src/github/client.ts | 37 ++++++++++++++++++++++ src/sync-manager.ts | 37 ++++++++++++++++++++-- 4 files changed, 145 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 68d2d89..7bd15e5 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@codemirror/view": "^6.36.2", "@uiw/react-codemirror": "^4.23.8", "codemirror": "^6.0.1", + "file-type": "^20.4.1", "react": "^19.0.0", "react-dom": "^19.0.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2099953..5ff754a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: codemirror: specifier: ^6.0.1 version: 6.0.1 + file-type: + specifier: ^20.4.1 + version: 20.4.1 react: specifier: ^19.0.0 version: 19.0.0 @@ -357,6 +360,13 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@tokenizer/inflate@0.2.7': + resolution: {integrity: sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==} + engines: {node: '>=18'} + + '@tokenizer/token@0.3.0': + resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + '@types/codemirror@5.60.8': resolution: {integrity: sha512-VjFgDF/eB+Aklcy15TtOTLQeMjTo07k7KAjql8OK5Dirr7a6sJY4T1uVBDuTVG9VEmn1uUsohOpYnVfgC6/jyw==} @@ -603,10 +613,17 @@ packages: fastq@1.18.0: resolution: {integrity: sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==} + fflate@0.8.2: + resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} + file-type@20.4.1: + resolution: {integrity: sha512-hw9gNZXUfZ02Jo0uafWLaFVPter5/k2rfcrjFJJHX/77xtSDOfJuEFb6oKlFV86FLP1SuyHMW1PSk0U9M5tKkQ==} + engines: {node: '>=18'} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -646,6 +663,9 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -760,6 +780,10 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + peek-readable@7.0.0: + resolution: {integrity: sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==} + engines: {node: '>=18'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -833,6 +857,10 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strtok3@10.2.2: + resolution: {integrity: sha512-Xt18+h4s7Z8xyZ0tmBoRmzxcop97R4BAh+dXouUDCYn+Em+1P3qpkUfI5ueWLT8ynC5hZ+q4iPEmGG1urvQGBg==} + engines: {node: '>=18'} + style-mod@4.1.2: resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==} @@ -844,6 +872,10 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + token-types@6.0.0: + resolution: {integrity: sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA==} + engines: {node: '>=14.16'} + ts-api-utils@1.4.3: resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==} engines: {node: '>=16'} @@ -862,6 +894,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + uint8array-extras@1.4.0: + resolution: {integrity: sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ==} + engines: {node: '>=18'} + undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} @@ -1180,6 +1216,16 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.18.0 + '@tokenizer/inflate@0.2.7': + dependencies: + debug: 4.4.0 + fflate: 0.8.2 + token-types: 6.0.0 + transitivePeerDependencies: + - supports-color + + '@tokenizer/token@0.3.0': {} + '@types/codemirror@5.60.8': dependencies: '@types/tern': 0.23.9 @@ -1505,10 +1551,21 @@ snapshots: dependencies: reusify: 1.0.4 + fflate@0.8.2: {} + file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 + file-type@20.4.1: + dependencies: + '@tokenizer/inflate': 0.2.7 + strtok3: 10.2.2 + token-types: 6.0.0 + uint8array-extras: 1.4.0 + transitivePeerDependencies: + - supports-color + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -1542,6 +1599,8 @@ snapshots: has-flag@4.0.0: {} + ieee754@1.2.1: {} + ignore@5.3.2: {} import-fresh@3.3.0: @@ -1646,6 +1705,8 @@ snapshots: path-key@3.1.1: {} + peek-readable@7.0.0: {} + picocolors@1.1.1: optional: true @@ -1698,6 +1759,11 @@ snapshots: strip-json-comments@3.1.1: {} + strtok3@10.2.2: + dependencies: + '@tokenizer/token': 0.3.0 + peek-readable: 7.0.0 + style-mod@4.1.2: {} supports-color@7.2.0: @@ -1708,6 +1774,11 @@ snapshots: dependencies: is-number: 7.0.0 + token-types@6.0.0: + dependencies: + '@tokenizer/token': 0.3.0 + ieee754: 1.2.1 + ts-api-utils@1.4.3(typescript@5.7.2): dependencies: typescript: 5.7.2 @@ -1720,6 +1791,8 @@ snapshots: typescript@5.7.2: {} + uint8array-extras@1.4.0: {} + undici-types@6.20.0: {} uri-js@4.4.1: diff --git a/src/github/client.ts b/src/github/client.ts index 591bb8e..9b24267 100644 --- a/src/github/client.ts +++ b/src/github/client.ts @@ -27,6 +27,13 @@ export type NewTreeRequestItem = { content?: string; }; +/** + * Response received when we create a new binary blob on GitHub + */ +export type CreatedBlob = { + sha: string; +}; + /** * Represents a git blob response from the GitHub API. */ @@ -170,6 +177,36 @@ export default class GithubClient { } } + /** + * Creates a new blob in the GitHub remote, this is mainly used to upload binary files. + * + * @param content The content of the blob to upload + * @param encoding Content encoding, can be "utf-8" or "base64". Defaults to "base64" + * @returns The SHA of the newly uploaded blob + */ + async createBlob( + content: string, + encoding: "utf-8" | "base64" = "base64", + ): Promise { + const res = await requestUrl({ + url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs`, + headers: this.headers(), + method: "POST", + body: JSON.stringify({ content, encoding }), + throw: false, + }); + if (res.status < 200 || res.status >= 400) { + await this.logger.error("Failed to create blob", res); + throw new GithubAPIError( + res.status, + `Failed to create blob, status ${res.status}`, + ); + } + return { + sha: res.json["sha"], + }; + } + /** * Gets a blob from its sha * @param url blob sha diff --git a/src/sync-manager.ts b/src/sync-manager.ts index 63fc99a..a439ae8 100644 --- a/src/sync-manager.ts +++ b/src/sync-manager.ts @@ -4,6 +4,7 @@ import { normalizePath, base64ToArrayBuffer, EventRef, + arrayBufferToBase64, } from "obsidian"; import GithubClient, { GetTreeResponseItem, @@ -20,6 +21,7 @@ import { GitHubSyncSettings } from "./settings/settings"; import Logger from "./logger"; import { decodeBase64String } from "./utils"; import GitHubSyncPlugin from "./main"; +import { fileTypeFromBuffer } from "file-type"; interface SyncAction { type: "upload" | "download" | "delete_local" | "delete_remote"; @@ -666,8 +668,8 @@ export default class SyncManager { * @returns String containing the file SHA1 */ async calculateSHA(filePath: string): Promise { - const content = await this.vault.adapter.read(filePath); - const contentBytes = new TextEncoder().encode(content); + const contentBuffer = await this.vault.adapter.readBinary(filePath); + const contentBytes = new Uint8Array(contentBuffer); const header = new TextEncoder().encode(`blob ${contentBytes.length}\0`); const store = new Uint8Array([...header, ...contentBytes]); return await crypto.subtle.digest("SHA-1", store).then((hash) => @@ -713,11 +715,40 @@ export default class SyncManager { // We don't save the metadata file after setting the SHAs cause we do that when // the sync is fully commited at the end. // TODO: Understand whether it's a problem we don't revert the SHA setting in case of sync failure + // + // In here we also upload blob is file is a binary. We do it here because when uploading a blob we + // also get back its SHA, so we can set it together with other files. + // We also do that right before creating the new tree because we need the SHAs of those blob to + // correctly create it. await Promise.all( Object.keys(treeFiles) .filter((filePath: string) => treeFiles[filePath].content) .map(async (filePath: string) => { - const newSha = await this.calculateSHA(filePath); + const buffer = await this.vault.adapter.readBinary(filePath); + const fileType = await fileTypeFromBuffer(buffer); + let newSha = ""; + if ( + // We can't determine the file type + fileType === undefined || + // This is not a text file + !fileType.mime.startsWith("text/") || + // Neither a json file + fileType.mime !== "application/json" + ) { + // We treat this file as a binary file. We can't upload these setting the content + // of a tree item, we first need to create a Git blob by uploading the file, then + // we must update the tree item to point the SHA to the blob we just created. + const hash = arrayBufferToBase64(buffer); + const { sha } = await this.client.createBlob(hash); + treeFiles[filePath].sha = sha; + // Can't have both sha and content set, so we delete it + delete treeFiles[filePath].content; + newSha = sha; + } else { + // File is text, we can upload the content directly + // so we just calculate the new SHA to keep track of it + newSha = await this.calculateSHA(filePath); + } this.metadataStore.data.files[filePath].sha = newSha; }), );