mirror of
https://github.com/silvanocerza/github-gitless-sync.git
synced 2026-07-22 05:41:36 +00:00
Simplify logic that determines if a file is text or binary
This commit is contained in:
Silvano Cerza
parent
5ab20cf482
commit
1a2eb64959
4 changed files with 40 additions and 109 deletions
|
|
@ -36,7 +36,6 @@
|
|||
"@uiw/react-codemirror": "^4.23.8",
|
||||
"@zip.js/zip.js": "^2.7.60",
|
||||
"codemirror": "^6.0.1",
|
||||
"file-type": "^20.4.1",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,9 +26,6 @@ importers:
|
|||
codemirror:
|
||||
specifier: ^6.0.1
|
||||
version: 6.0.1
|
||||
file-type:
|
||||
specifier: ^20.4.1
|
||||
version: 20.4.1
|
||||
react:
|
||||
specifier: ^19.0.0
|
||||
version: 19.0.0
|
||||
|
|
@ -537,13 +534,6 @@ packages:
|
|||
'@sinonjs/samsam@8.0.2':
|
||||
resolution: {integrity: sha512-v46t/fwnhejRSFTGqbpn9u+LQ9xJDse10gNnPgAcxgdoCDMXj/G2asWAC/8Qs+BAZDicX+MNZouXT1A7c83kVw==}
|
||||
|
||||
'@tokenizer/inflate@0.2.7':
|
||||
resolution: {integrity: sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@tokenizer/token@0.3.0':
|
||||
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
|
||||
|
||||
'@types/codemirror@5.60.8':
|
||||
resolution: {integrity: sha512-VjFgDF/eB+Aklcy15TtOTLQeMjTo07k7KAjql8OK5Dirr7a6sJY4T1uVBDuTVG9VEmn1uUsohOpYnVfgC6/jyw==}
|
||||
|
||||
|
|
@ -812,17 +802,10 @@ packages:
|
|||
fastq@1.18.0:
|
||||
resolution: {integrity: sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==}
|
||||
|
||||
fflate@0.8.2:
|
||||
resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
|
||||
|
||||
file-entry-cache@8.0.0:
|
||||
resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
|
||||
file-type@20.4.1:
|
||||
resolution: {integrity: sha512-hw9gNZXUfZ02Jo0uafWLaFVPter5/k2rfcrjFJJHX/77xtSDOfJuEFb6oKlFV86FLP1SuyHMW1PSk0U9M5tKkQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
fill-keys@1.0.2:
|
||||
resolution: {integrity: sha512-tcgI872xXjwFF4xgQmLxi76GnwJG3g/3isB1l4/G5Z4zrbddGpBjqZCO9oEAcB5wX0Hj/5iQB3toxfO7in1hHA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
|
@ -876,9 +859,6 @@ packages:
|
|||
resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
ignore@5.3.2:
|
||||
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
|
||||
engines: {node: '>= 4'}
|
||||
|
|
@ -1013,10 +993,6 @@ packages:
|
|||
path-parse@1.0.7:
|
||||
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
|
||||
|
||||
peek-readable@7.0.0:
|
||||
resolution: {integrity: sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
picocolors@1.1.1:
|
||||
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
|
||||
|
||||
|
|
@ -1104,10 +1080,6 @@ packages:
|
|||
resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
strtok3@10.2.2:
|
||||
resolution: {integrity: sha512-Xt18+h4s7Z8xyZ0tmBoRmzxcop97R4BAh+dXouUDCYn+Em+1P3qpkUfI5ueWLT8ynC5hZ+q4iPEmGG1urvQGBg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
style-mod@4.1.2:
|
||||
resolution: {integrity: sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==}
|
||||
|
||||
|
|
@ -1123,10 +1095,6 @@ packages:
|
|||
resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
|
||||
engines: {node: '>=8.0'}
|
||||
|
||||
token-types@6.0.0:
|
||||
resolution: {integrity: sha512-lbDrTLVsHhOMljPscd0yitpozq7Ga2M5Cvez5AjGg8GASBjtt6iERCAJ93yommPmz62fb45oFIXHEZ3u9bfJEA==}
|
||||
engines: {node: '>=14.16'}
|
||||
|
||||
ts-api-utils@1.4.3:
|
||||
resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==}
|
||||
engines: {node: '>=16'}
|
||||
|
|
@ -1158,10 +1126,6 @@ packages:
|
|||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
uint8array-extras@1.4.0:
|
||||
resolution: {integrity: sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
undici-types@6.20.0:
|
||||
resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
|
||||
|
||||
|
|
@ -1569,16 +1533,6 @@ snapshots:
|
|||
lodash.get: 4.4.2
|
||||
type-detect: 4.1.0
|
||||
|
||||
'@tokenizer/inflate@0.2.7':
|
||||
dependencies:
|
||||
debug: 4.4.0
|
||||
fflate: 0.8.2
|
||||
token-types: 6.0.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@tokenizer/token@0.3.0': {}
|
||||
|
||||
'@types/codemirror@5.60.8':
|
||||
dependencies:
|
||||
'@types/tern': 0.23.9
|
||||
|
|
@ -1944,21 +1898,10 @@ snapshots:
|
|||
dependencies:
|
||||
reusify: 1.0.4
|
||||
|
||||
fflate@0.8.2: {}
|
||||
|
||||
file-entry-cache@8.0.0:
|
||||
dependencies:
|
||||
flat-cache: 4.0.1
|
||||
|
||||
file-type@20.4.1:
|
||||
dependencies:
|
||||
'@tokenizer/inflate': 0.2.7
|
||||
strtok3: 10.2.2
|
||||
token-types: 6.0.0
|
||||
uint8array-extras: 1.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
fill-keys@1.0.2:
|
||||
dependencies:
|
||||
is-object: 1.0.2
|
||||
|
|
@ -2007,8 +1950,6 @@ snapshots:
|
|||
dependencies:
|
||||
function-bind: 1.1.2
|
||||
|
||||
ieee754@1.2.1: {}
|
||||
|
||||
ignore@5.3.2: {}
|
||||
|
||||
import-fresh@3.3.0:
|
||||
|
|
@ -2127,8 +2068,6 @@ snapshots:
|
|||
|
||||
path-parse@1.0.7: {}
|
||||
|
||||
peek-readable@7.0.0: {}
|
||||
|
||||
picocolors@1.1.1:
|
||||
optional: true
|
||||
|
||||
|
|
@ -2203,11 +2142,6 @@ snapshots:
|
|||
|
||||
strip-json-comments@3.1.1: {}
|
||||
|
||||
strtok3@10.2.2:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
peek-readable: 7.0.0
|
||||
|
||||
style-mod@4.1.2: {}
|
||||
|
||||
supports-color@7.2.0:
|
||||
|
|
@ -2220,11 +2154,6 @@ snapshots:
|
|||
dependencies:
|
||||
is-number: 7.0.0
|
||||
|
||||
token-types@6.0.0:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
ieee754: 1.2.1
|
||||
|
||||
ts-api-utils@1.4.3(typescript@5.7.2):
|
||||
dependencies:
|
||||
typescript: 5.7.2
|
||||
|
|
@ -2248,8 +2177,6 @@ snapshots:
|
|||
|
||||
typescript@5.7.2: {}
|
||||
|
||||
uint8array-extras@1.4.0: {}
|
||||
|
||||
undici-types@6.20.0: {}
|
||||
|
||||
uri-js@4.4.1:
|
||||
|
|
|
|||
|
|
@ -18,9 +18,8 @@ import MetadataStore, {
|
|||
import EventsListener from "./events-listener";
|
||||
import { GitHubSyncSettings } from "./settings/settings";
|
||||
import Logger from "./logger";
|
||||
import { decodeBase64String } from "./utils";
|
||||
import { decodeBase64String, hasTextExtension } from "./utils";
|
||||
import GitHubSyncPlugin from "./main";
|
||||
import { fileTypeFromBuffer } from "file-type";
|
||||
import { BlobReader, Entry, Uint8ArrayWriter, ZipReader } from "@zip.js/zip.js";
|
||||
|
||||
interface SyncAction {
|
||||
|
|
@ -742,45 +741,25 @@ export default class SyncManager {
|
|||
Object.keys(treeFiles)
|
||||
.filter((filePath: string) => treeFiles[filePath].content)
|
||||
.map(async (filePath: string) => {
|
||||
// Some Markdown or JSON files might grow to be quite big, that makes it
|
||||
// impossible for the file-type library to guess their file type.
|
||||
// It also increases the amount of memory used by A LOT and might cause
|
||||
// issues in devices with low memory if there are lots of files to check.
|
||||
//
|
||||
// I don't fully trust file extensions as they're not completely reliable
|
||||
// to determine the file type, though I feel it's ok to compromise and rely
|
||||
// on them if it makes the plugin handle upload better on certain devices.
|
||||
if (filePath.endsWith(".md") || filePath.endsWith(".json")) {
|
||||
this.metadataStore.data.files[filePath].sha =
|
||||
await this.calculateSHA(filePath);
|
||||
if (hasTextExtension(filePath)) {
|
||||
const sha = await this.calculateSHA(filePath);
|
||||
this.metadataStore.data.files[filePath].sha = sha;
|
||||
return;
|
||||
}
|
||||
|
||||
// We can't upload binary files by setting the content of a tree item,
|
||||
// we first need to create a Git blob by uploading the file, then
|
||||
// we must update the tree item to point the SHA to the blob we just created.
|
||||
const buffer = await this.vault.adapter.readBinary(filePath);
|
||||
const fileType = await fileTypeFromBuffer(buffer);
|
||||
let newSha = "";
|
||||
if (
|
||||
// We can't determine the file type
|
||||
fileType === undefined ||
|
||||
// This is not a text file
|
||||
!fileType.mime.startsWith("text/") ||
|
||||
// Neither a json file
|
||||
fileType.mime !== "application/json"
|
||||
) {
|
||||
// We treat this file as a binary file. We can't upload these setting the content
|
||||
// of a tree item, we first need to create a Git blob by uploading the file, then
|
||||
// we must update the tree item to point the SHA to the blob we just created.
|
||||
const hash = arrayBufferToBase64(buffer);
|
||||
const { sha } = await this.client.createBlob(hash);
|
||||
treeFiles[filePath].sha = sha;
|
||||
// Can't have both sha and content set, so we delete it
|
||||
delete treeFiles[filePath].content;
|
||||
newSha = sha;
|
||||
} else {
|
||||
// File is text, we can upload the content directly
|
||||
// so we just calculate the new SHA to keep track of it
|
||||
newSha = await this.calculateSHA(filePath);
|
||||
}
|
||||
this.metadataStore.data.files[filePath].sha = newSha;
|
||||
const hash = arrayBufferToBase64(buffer);
|
||||
const { sha } = await this.client.createBlob(hash);
|
||||
treeFiles[filePath].sha = sha;
|
||||
// Can't have both sha and content set, so we delete it
|
||||
delete treeFiles[filePath].content;
|
||||
this.metadataStore.data.files[filePath].sha = sha;
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
26
src/utils.ts
26
src/utils.ts
|
|
@ -1,5 +1,15 @@
|
|||
import { base64ToArrayBuffer } from "obsidian";
|
||||
|
||||
const TEXT_EXTENSIONS = [
|
||||
".css",
|
||||
".md",
|
||||
".json",
|
||||
".txt",
|
||||
".csv",
|
||||
".js",
|
||||
".log",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Decodes a base64 encoded string, this properly
|
||||
* handles emojis and other non ASCII chars.
|
||||
|
|
@ -37,3 +47,19 @@ export async function copyToClipboard(text: string) {
|
|||
document.body.removeChild(textarea);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a file path has one of the predefined text extensions.
|
||||
* This is a best guess at best.
|
||||
*
|
||||
* @param filePath The path of the file to check
|
||||
* @returns True if the file has a text extension, false otherwise
|
||||
*/
|
||||
export function hasTextExtension(filePath: string) {
|
||||
for (const extension in TEXT_EXTENSIONS) {
|
||||
if (filePath.endsWith(extension)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue