mirror of
https://github.com/silvanocerza/github-gitless-sync.git
synced 2026-07-22 05:41:36 +00:00
Fix binary files failing to upload to GitHub
This commit is contained in:
Silvano Cerza
parent
c1d1db6df3
commit
5f0c8b1ed2
4 changed files with 145 additions and 3 deletions
|
|
@ -29,6 +29,7 @@
|
|||
"@codemirror/view": "^6.36.2",
|
||||
"@uiw/react-codemirror": "^4.23.8",
|
||||
"codemirror": "^6.0.1",
|
||||
"file-type": "^20.4.1",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ importers:
|
|||
codemirror:
|
||||
specifier: ^6.0.1
|
||||
version: 6.0.1
|
||||
file-type:
|
||||
specifier: ^20.4.1
|
||||
version: 20.4.1
|
||||
react:
|
||||
specifier: ^19.0.0
|
||||
version: 19.0.0
|
||||
|
|
@ -357,6 +360,13 @@ packages:
|
|||
resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
'@tokenizer/inflate@0.2.7':
|
||||
resolution: {integrity: sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@tokenizer/token@0.3.0':
|
||||
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
|
||||
|
||||
'@types/codemirror@5.60.8':
|
||||
resolution: {integrity: sha512-VjFgDF/eB+Aklcy15TtOTLQeMjTo07k7KAjql8OK5Dirr7a6sJY4T1uVBDuTVG9VEmn1uUsohOpYnVfgC6/jyw==}
|
||||
|
||||
|
|
@ -603,10 +613,17 @@ packages:
|
|||
fastq@1.18.0:
|
||||
resolution: {integrity: sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==}
|
||||
|
||||
fflate@0.8.2:
|
||||
resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
|
||||
|
||||
file-entry-cache@8.0.0:
|
||||
resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
|
||||
file-type@20.4.1:
|
||||
resolution: {integrity: sha512-hw9gNZXUfZ02Jo0uafWLaFVPter5/k2rfcrjFJJHX/77xtSDOfJuEFb6oKlFV86FLP1SuyHMW1PSk0U9M5tKkQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
fill-range@7.1.1:
|
||||
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -646,6 +663,9 @@ packages:
|
|||
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
ignore@5.3.2:
|
||||
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
|
||||
engines: {node: '>= 4'}
|
||||
|
|
@ -760,6 +780,10 @@ packages:
|
|||
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
peek-readable@7.0.0:
|
||||
resolution: {integrity: sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
picocolors@1.1.1:
|
||||
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
|
||||
|
||||
|
|
@ -833,6 +857,10 @@ packages:
|
|||
resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
strtok3@10.2.2:
|
||||
resolution: {integrity: sha512-Xt18+h4s7Z8xyZ0tmBoRmzxcop97R4BAh+dXouUDCYn+Em+1P3qpkUfI5ueWLT8ynC5hZ+q4iPEmGG1urvQGBg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
style-mod@4.1.2:
|
||||
resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==}
|
||||
|
||||
|
|
@ -844,6 +872,10 @@ packages:
|
|||
resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
|
||||
engines: {node: '>=8.0'}
|
||||
|
||||
token-types@6.0.0:
|
||||
resolution: {integrity: sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA==}
|
||||
engines: {node: '>=14.16'}
|
||||
|
||||
ts-api-utils@1.4.3:
|
||||
resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==}
|
||||
engines: {node: '>=16'}
|
||||
|
|
@ -862,6 +894,10 @@ packages:
|
|||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
uint8array-extras@1.4.0:
|
||||
resolution: {integrity: sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
undici-types@6.20.0:
|
||||
resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
|
||||
|
||||
|
|
@ -1180,6 +1216,16 @@ snapshots:
|
|||
'@nodelib/fs.scandir': 2.1.5
|
||||
fastq: 1.18.0
|
||||
|
||||
'@tokenizer/inflate@0.2.7':
|
||||
dependencies:
|
||||
debug: 4.4.0
|
||||
fflate: 0.8.2
|
||||
token-types: 6.0.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@tokenizer/token@0.3.0': {}
|
||||
|
||||
'@types/codemirror@5.60.8':
|
||||
dependencies:
|
||||
'@types/tern': 0.23.9
|
||||
|
|
@ -1505,10 +1551,21 @@ snapshots:
|
|||
dependencies:
|
||||
reusify: 1.0.4
|
||||
|
||||
fflate@0.8.2: {}
|
||||
|
||||
file-entry-cache@8.0.0:
|
||||
dependencies:
|
||||
flat-cache: 4.0.1
|
||||
|
||||
file-type@20.4.1:
|
||||
dependencies:
|
||||
'@tokenizer/inflate': 0.2.7
|
||||
strtok3: 10.2.2
|
||||
token-types: 6.0.0
|
||||
uint8array-extras: 1.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
fill-range@7.1.1:
|
||||
dependencies:
|
||||
to-regex-range: 5.0.1
|
||||
|
|
@ -1542,6 +1599,8 @@ snapshots:
|
|||
|
||||
has-flag@4.0.0: {}
|
||||
|
||||
ieee754@1.2.1: {}
|
||||
|
||||
ignore@5.3.2: {}
|
||||
|
||||
import-fresh@3.3.0:
|
||||
|
|
@ -1646,6 +1705,8 @@ snapshots:
|
|||
|
||||
path-key@3.1.1: {}
|
||||
|
||||
peek-readable@7.0.0: {}
|
||||
|
||||
picocolors@1.1.1:
|
||||
optional: true
|
||||
|
||||
|
|
@ -1698,6 +1759,11 @@ snapshots:
|
|||
|
||||
strip-json-comments@3.1.1: {}
|
||||
|
||||
strtok3@10.2.2:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
peek-readable: 7.0.0
|
||||
|
||||
style-mod@4.1.2: {}
|
||||
|
||||
supports-color@7.2.0:
|
||||
|
|
@ -1708,6 +1774,11 @@ snapshots:
|
|||
dependencies:
|
||||
is-number: 7.0.0
|
||||
|
||||
token-types@6.0.0:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
ieee754: 1.2.1
|
||||
|
||||
ts-api-utils@1.4.3(typescript@5.7.2):
|
||||
dependencies:
|
||||
typescript: 5.7.2
|
||||
|
|
@ -1720,6 +1791,8 @@ snapshots:
|
|||
|
||||
typescript@5.7.2: {}
|
||||
|
||||
uint8array-extras@1.4.0: {}
|
||||
|
||||
undici-types@6.20.0: {}
|
||||
|
||||
uri-js@4.4.1:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,13 @@ export type NewTreeRequestItem = {
|
|||
content?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Response received when we create a new binary blob on GitHub
|
||||
*/
|
||||
export type CreatedBlob = {
|
||||
sha: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Represents a git blob response from the GitHub API.
|
||||
*/
|
||||
|
|
@ -170,6 +177,36 @@ export default class GithubClient {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new blob in the GitHub remote, this is mainly used to upload binary files.
|
||||
*
|
||||
* @param content The content of the blob to upload
|
||||
* @param encoding Content encoding, can be "utf-8" or "base64". Defaults to "base64"
|
||||
* @returns The SHA of the newly uploaded blob
|
||||
*/
|
||||
async createBlob(
|
||||
content: string,
|
||||
encoding: "utf-8" | "base64" = "base64",
|
||||
): Promise<CreatedBlob> {
|
||||
const res = await requestUrl({
|
||||
url: `https://api.github.com/repos/${this.settings.githubOwner}/${this.settings.githubRepo}/git/blobs`,
|
||||
headers: this.headers(),
|
||||
method: "POST",
|
||||
body: JSON.stringify({ content, encoding }),
|
||||
throw: false,
|
||||
});
|
||||
if (res.status < 200 || res.status >= 400) {
|
||||
await this.logger.error("Failed to create blob", res);
|
||||
throw new GithubAPIError(
|
||||
res.status,
|
||||
`Failed to create blob, status ${res.status}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
sha: res.json["sha"],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a blob from its sha
|
||||
* @param url blob sha
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
normalizePath,
|
||||
base64ToArrayBuffer,
|
||||
EventRef,
|
||||
arrayBufferToBase64,
|
||||
} from "obsidian";
|
||||
import GithubClient, {
|
||||
GetTreeResponseItem,
|
||||
|
|
@ -20,6 +21,7 @@ import { GitHubSyncSettings } from "./settings/settings";
|
|||
import Logger from "./logger";
|
||||
import { decodeBase64String } from "./utils";
|
||||
import GitHubSyncPlugin from "./main";
|
||||
import { fileTypeFromBuffer } from "file-type";
|
||||
|
||||
interface SyncAction {
|
||||
type: "upload" | "download" | "delete_local" | "delete_remote";
|
||||
|
|
@ -666,8 +668,8 @@ export default class SyncManager {
|
|||
* @returns String containing the file SHA1
|
||||
*/
|
||||
async calculateSHA(filePath: string): Promise<string> {
|
||||
const content = await this.vault.adapter.read(filePath);
|
||||
const contentBytes = new TextEncoder().encode(content);
|
||||
const contentBuffer = await this.vault.adapter.readBinary(filePath);
|
||||
const contentBytes = new Uint8Array(contentBuffer);
|
||||
const header = new TextEncoder().encode(`blob ${contentBytes.length}\0`);
|
||||
const store = new Uint8Array([...header, ...contentBytes]);
|
||||
return await crypto.subtle.digest("SHA-1", store).then((hash) =>
|
||||
|
|
@ -713,11 +715,40 @@ export default class SyncManager {
|
|||
// We don't save the metadata file after setting the SHAs cause we do that when
|
||||
// the sync is fully commited at the end.
|
||||
// TODO: Understand whether it's a problem we don't revert the SHA setting in case of sync failure
|
||||
//
|
||||
// In here we also upload blob is file is a binary. We do it here because when uploading a blob we
|
||||
// also get back its SHA, so we can set it together with other files.
|
||||
// We also do that right before creating the new tree because we need the SHAs of those blob to
|
||||
// correctly create it.
|
||||
await Promise.all(
|
||||
Object.keys(treeFiles)
|
||||
.filter((filePath: string) => treeFiles[filePath].content)
|
||||
.map(async (filePath: string) => {
|
||||
const newSha = await this.calculateSHA(filePath);
|
||||
const buffer = await this.vault.adapter.readBinary(filePath);
|
||||
const fileType = await fileTypeFromBuffer(buffer);
|
||||
let newSha = "";
|
||||
if (
|
||||
// We can't determine the file type
|
||||
fileType === undefined ||
|
||||
// This is not a text file
|
||||
!fileType.mime.startsWith("text/") ||
|
||||
// Neither a json file
|
||||
fileType.mime !== "application/json"
|
||||
) {
|
||||
// We treat this file as a binary file. We can't upload these setting the content
|
||||
// of a tree item, we first need to create a Git blob by uploading the file, then
|
||||
// we must update the tree item to point the SHA to the blob we just created.
|
||||
const hash = arrayBufferToBase64(buffer);
|
||||
const { sha } = await this.client.createBlob(hash);
|
||||
treeFiles[filePath].sha = sha;
|
||||
// Can't have both sha and content set, so we delete it
|
||||
delete treeFiles[filePath].content;
|
||||
newSha = sha;
|
||||
} else {
|
||||
// File is text, we can upload the content directly
|
||||
// so we just calculate the new SHA to keep track of it
|
||||
newSha = await this.calculateSHA(filePath);
|
||||
}
|
||||
this.metadataStore.data.files[filePath].sha = newSha;
|
||||
}),
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in a new issue