Ditch partial sync

This commit is contained in:
Silvano Cerza 2025-01-23 11:09:58 +01:00
parent 923a811a4a
commit d20bda9a68
8 changed files with 63 additions and 272 deletions

View file

@ -12,8 +12,6 @@ These are the main features of the plugin:
- Desktop and mobile support
- Doesn't require `git`
- Multiple vaults sync
- Partial vault sync
- Partial remote repository sync
- Automatic sync on fixed interval
- Manual sync
@ -63,15 +61,6 @@ If you don't want to see the button you can hide it, just check the plugin setti
The `Sync with GitHub` command is also available.
### Partial sync
Optionally you can choose to sync only a part of your vault and/or your repository.
This might be useful if you keep the sources for your blog in your vault as example, though you sync with some other service.
> [!WARNING]
> For the time being changing the directories after onboarding is not supported.
> I mean, you can do it, though I have not tested it and it's likely to break in unforeseeable ways.
### Config sync
If you want to sync your vault configs with other vault you can enable that.

View file

@ -38,23 +38,8 @@ export default class EventsListener {
return;
}
let remotePath: string;
if (file.path.startsWith(this.settings.localContentDir)) {
remotePath = file.path.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
} else if (
file.path.startsWith(`${this.vault.configDir}/github-sync-metadata.json`)
) {
remotePath = file.path;
} else {
throw new Error("Unexpected file path");
}
this.metadataStore.data.files[file.path] = {
localPath: file.path,
remotePath: remotePath!,
path: file.path,
sha: null,
dirty: true,
// This file has been created by the user
@ -132,9 +117,12 @@ export default class EventsListener {
private isSyncable(filePath: string) {
return (
filePath.startsWith(this.settings.localContentDir) ||
filePath === `${this.vault.configDir}/github-sync-metadata.json` ||
(this.settings.syncConfigDir && filePath.startsWith(this.vault.configDir))
(this.settings.syncConfigDir &&
filePath.startsWith(this.vault.configDir)) ||
// Obsidian recommends not syncing the workspace files
filePath === `${this.vault.configDir}/workspace.json` ||
filePath === `${this.vault.configDir}/workspace-mobile.json`
);
}
}

View file

@ -43,8 +43,6 @@ export default class GithubClient {
private owner: string,
private repo: string,
private branch: string,
private repoContentDir: string,
private configDir: string,
) {}
headers() {

View file

@ -6,9 +6,7 @@ import { Vault, normalizePath } from "obsidian";
*/
export interface FileMetadata {
// Local path to the file
localPath: string;
// Path to the file in the remote repository.
remotePath: string;
path: string;
// SHA of the file in the remote repository.
// This is necessary to update the file remotely.
// If this is null the file has not yet been pushed to the remote repository.

View file

@ -4,8 +4,6 @@ export interface GitHubSyncSettings {
githubOwner: string;
githubRepo: string;
githubBranch: string;
repoContentDir: string;
localContentDir: string;
syncStrategy: "manual" | "interval";
syncInterval: number;
syncOnStartup: boolean;
@ -21,8 +19,6 @@ export const DEFAULT_SETTINGS: GitHubSyncSettings = {
githubOwner: "",
githubRepo: "",
githubBranch: "main",
repoContentDir: "",
localContentDir: "",
syncStrategy: "manual",
syncInterval: 1,
syncOnStartup: false,

View file

@ -85,42 +85,6 @@ export default class GitHubSyncSettingsTab extends PluginSettingTab {
}),
);
containerEl.createEl("h2", { text: "Folders" });
new Setting(containerEl)
.setName("Repository content directory")
.setDesc(
`The repository directory to sync, relative to the repository root.
If not set the whole repository will be synced.`,
)
.addText((text) =>
text
.setPlaceholder("Exaple: blog/content")
.setValue(this.plugin.settings.repoContentDir)
.onChange(async (value) => {
// TODO: Change the local path if already fetched
this.plugin.settings.repoContentDir = value;
await this.plugin.saveSettings();
}),
);
new Setting(containerEl)
.setName("Local content directory")
.setDesc(
`The local directory to sync, relative to the vault root.
If not set the whole vault will be synced.`,
)
.addText((text) =>
text
.setPlaceholder("Exaple: folder/blog-posts")
.setValue(this.plugin.settings.localContentDir)
.onChange(async (value) => {
// TODO: Move the folder if already fetched
this.plugin.settings.localContentDir = value;
await this.plugin.saveSettings();
}),
);
containerEl.createEl("h2", { text: "Sync" });
const syncStrategies = {

View file

@ -34,8 +34,6 @@ export default class SyncManager {
this.settings.githubOwner,
this.settings.githubRepo,
this.settings.githubBranch,
this.settings.repoContentDir,
this.vault.configDir,
);
this.eventsListener = new EventsListener(
this.vault,
@ -45,45 +43,23 @@ export default class SyncManager {
}
/**
* Returns true if the remote content dir is empty.
* @param files All files in the remote repository
* Returns true if the local vault root is empty.
*/
private remoteContentDirIsEmpty(files: {
[key: string]: GetTreeResponseItem;
}): boolean {
private async vaultIsEmpty(): Promise<boolean> {
const { files, folders } = await this.vault.adapter.list(
this.vault.getRoot().path,
);
// There are files or folders in the vault dir
return (
Object.keys(files).filter((filePath: string) =>
filePath.startsWith(this.settings.repoContentDir),
).length === 0
files.length === 0 ||
// We filter out the config dir since is always present so it's fine if we find it.
folders.filter((f) => f !== this.vault.configDir).length === 0
);
}
/**
* Returns true if the local content dir is empty.
* If the local content dir is the vault root the config dir is ignored.
*/
private async localContentDirIsEmpty(): Promise<boolean> {
const localContentDirExists = await this.vault.adapter.exists(
this.settings.localContentDir,
);
if (localContentDirExists) {
const { files, folders } = await this.vault.adapter.list(
this.settings.localContentDir,
);
// There are files or folders in the local content dir
return (
files.length === 0 ||
// We filter out the config dir in case the user wants to sync the whole
// vault. The config dir is always present so it's fine if we find it.
folders.filter((f) => f !== this.vault.configDir).length === 0
);
}
return true;
}
/**
* Handles first sync with remote and local.
* This fails neither remote nor local folders are empty.
* This fails if neither remote nor local folders are empty.
*/
async firstSync() {
let repositoryIsBare = false;
@ -122,21 +98,21 @@ export default class SyncManager {
treeSha = res.sha;
}
const remoteContentDirIsEmpty = this.remoteContentDirIsEmpty(files);
const localContentDirIsEmpty = await this.localContentDirIsEmpty();
const remoteRepoIsEmpty = Object.keys(files).length === 0;
const vaultIsEmpty = await this.vaultIsEmpty();
if (!remoteContentDirIsEmpty && !localContentDirIsEmpty) {
if (!remoteRepoIsEmpty && !vaultIsEmpty) {
// Both have files, we can't sync, show error
throw new Error("Both remote and local have files, can't sync");
} else if (remoteContentDirIsEmpty || repositoryIsBare) {
} else if (remoteRepoIsEmpty || repositoryIsBare) {
// Remote has no files and no manifest, let's just upload whatever we have locally.
// This is fine even if the local content dir is empty.
// This is fine even if the vault is empty.
// The most important thing at this point is that the remote manifest is created.
await this.firstSyncFromLocal(files, treeSha);
} else {
// Local has no files and there's no manifest in the remote repo.
// Let's download whatever we have in the remote content dir.
// This is fine even if the remote content dir is empty.
// Let's download whatever we have in the remote repo.
// This is fine even if the remote repo is empty.
// In this case too the important step is that the remote manifest is created.
await this.firstSyncFromRemote(files, treeSha);
}
@ -157,21 +133,16 @@ export default class SyncManager {
await Promise.all(
Object.keys(files)
.filter((filePath: string) => {
if (filePath.startsWith(this.settings.repoContentDir)) {
return true;
} else if (
filePath === `${this.vault.configDir}/github-sync-metadata.json`
) {
// The metadata file must always be synced
return true;
} else if (
if (
this.settings.syncConfigDir &&
filePath.startsWith(this.vault.configDir)
filePath.startsWith(this.vault.configDir) &&
filePath !== `${this.vault.configDir}/github-sync-metadata.json`
) {
// Include files in the config dir only if the user has enabled it
return true;
// Include files in the config dir only if the user has enabled it.
// The metadata file must always be synced.
return false;
}
return false;
return true;
})
.map(async (filePath: string) => {
await this.downloadFile(files[filePath], Date.now());
@ -197,9 +168,8 @@ export default class SyncManager {
async (filePath: string) => {
const normalizedPath = normalizePath(filePath);
const content = await this.vault.adapter.read(normalizedPath);
const { remotePath } = this.metadataStore.data.files[filePath];
newTreeFiles[remotePath] = {
path: remotePath,
newTreeFiles[filePath] = {
path: filePath,
mode: "100644",
type: "blob",
content: content,
@ -212,10 +182,10 @@ export default class SyncManager {
/**
* Handles first sync with the remote repository.
* This must be called in case there are no files in the remote content dir and no manifest while
* local content dir has files and a manifest.
* This must be called in case there are no files in the remote repo and no manifest while
* local vault has files and a manifest.
*
* @param files All files in the remote repository, including those not in its content dir.
* @param files All files in the remote repository
* @param treeSha The SHA of the tree in the remote repository.
*/
async firstSyncFromLocal(
@ -241,9 +211,8 @@ export default class SyncManager {
async (filePath: string) => {
const normalizedPath = normalizePath(filePath);
const content = await this.vault.adapter.read(normalizedPath);
const { remotePath } = this.metadataStore.data.files[filePath];
newTreeFiles[remotePath] = {
path: remotePath,
newTreeFiles[filePath] = {
path: filePath,
mode: "100644",
type: "blob",
content: content,
@ -286,12 +255,12 @@ export default class SyncManager {
if (resolution) {
conflictResolutions.push({
type: "download",
filePath: conflicts[index].remoteFile.localPath,
filePath: conflicts[index].remoteFile.path,
});
} else {
conflictResolutions.push({
type: "upload",
filePath: conflicts[index].localFile.localPath,
filePath: conflicts[index].localFile.path,
});
}
},
@ -334,12 +303,8 @@ export default class SyncManager {
case "upload": {
const normalizedPath = normalizePath(action.filePath);
const content = await this.vault.adapter.read(normalizedPath);
const remotePath = action.filePath.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
newTreeFiles[remotePath] = {
path: remotePath,
newTreeFiles[action.filePath] = {
path: action.filePath,
mode: "100644",
type: "blob",
content: content,
@ -347,9 +312,7 @@ export default class SyncManager {
break;
}
case "delete_remote": {
const { remotePath } =
this.metadataStore.data.files[action.filePath];
newTreeFiles[remotePath].sha = null;
newTreeFiles[action.filePath].sha = null;
break;
}
case "download":
@ -365,12 +328,8 @@ export default class SyncManager {
...actions
.filter((action) => action.type === "download")
.map(async (action: SyncAction) => {
const remotePath = action.filePath.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
await this.downloadFile(
files[remotePath],
files[action.filePath],
remoteMetadata.files[action.filePath].lastModified,
);
}),
@ -386,8 +345,8 @@ export default class SyncManager {
/**
* Finds conflicts between local and remote files.
* @param remoteFiles All files in the remote content dir
* @param localFiles All files in the local content dir
* @param remoteFiles All files in the remote repo
* @param localFiles All files in the local vault
* @returns List of objects with both remote and local conflicting files metadata
*/
findConflicts(
@ -432,8 +391,8 @@ export default class SyncManager {
/**
* Determines which sync action to take for each file.
*
* @param remoteFiles All files in the remote content dir
* @param localFiles All files in the local content dir
* @param remoteFiles All files in the remote repo
* @param localFiles All files in the local vault
*
* @returns List of SyncActions
*/
@ -589,30 +548,26 @@ export default class SyncManager {
async downloadFile(file: GetTreeResponseItem, lastModified: number) {
const url = file.url;
const destinationFile = file.path.replace(
this.settings.repoContentDir,
this.settings.localContentDir,
);
const fileMetadata = this.metadataStore.data.files[destinationFile];
const fileMetadata = this.metadataStore.data.files[file.path];
if (fileMetadata && fileMetadata.sha === file.sha) {
// File already exists and has the same SHA, no need to download it again.
return;
}
const blob = await this.client.getBlob(url);
const destinationFolder = normalizePath(
destinationFile.split("/").slice(0, -1).join("/"),
const normalizedPath = normalizePath(file.path);
const fileFolder = normalizePath(
normalizedPath.split("/").slice(0, -1).join("/"),
);
if (!(await this.vault.adapter.exists(destinationFolder))) {
await this.vault.adapter.mkdir(destinationFolder);
if (!(await this.vault.adapter.exists(fileFolder))) {
await this.vault.adapter.mkdir(fileFolder);
}
this.vault.adapter.writeBinary(
normalizePath(destinationFile),
normalizedPath,
Buffer.from(blob.content, "base64"),
);
this.metadataStore.data.files[destinationFile] = {
localPath: destinationFile,
remotePath: file.path,
this.metadataStore.data.files[file.path] = {
path: file.path,
sha: file.sha,
dirty: false,
justDownloaded: true,
@ -633,7 +588,7 @@ export default class SyncManager {
await this.metadataStore.load();
if (Object.keys(this.metadataStore.data.files).length === 0) {
let files = [];
let folders = [this.settings.localContentDir];
let folders = [this.vault.getRoot().path];
while (folders.length > 0) {
const folder = folders.pop();
if (folder === undefined) {
@ -653,28 +608,8 @@ export default class SyncManager {
return;
}
// We change the remote path only if the file is not in the config dir
// as that directory is always at the root of the repo.
// If it's synced obviously.
let remotePath = filePath;
if (!filePath.startsWith(this.vault.configDir)) {
if (this.settings.localContentDir === "") {
if (this.settings.repoContentDir === "") {
remotePath = filePath;
} else {
remotePath = `${this.settings.repoContentDir}/${filePath}`;
}
} else {
remotePath = filePath.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
}
}
this.metadataStore.data.files[filePath] = {
localPath: filePath,
remotePath: remotePath,
path: filePath,
sha: null,
dirty: false,
justDownloaded: false,
@ -683,12 +618,11 @@ export default class SyncManager {
});
// Must be the first time we run, initialize the metadata store
// with itself and all files in the local content dir.
// with itself and all files in the vault.
this.metadataStore.data.files[
`${this.vault.configDir}/github-sync-metadata.json`
] = {
localPath: `${this.vault.configDir}/github-sync-metadata.json`,
remotePath: `${this.vault.configDir}/github-sync-metadata.json`,
path: `${this.vault.configDir}/github-sync-metadata.json`,
sha: null,
dirty: false,
justDownloaded: false,
@ -719,9 +653,7 @@ export default class SyncManager {
// Add them to the metadata store
files.forEach((filePath: string) => {
this.metadataStore.data.files[filePath] = {
localPath: filePath,
// Remote path is the same for config dir files
remotePath: filePath,
path: filePath,
sha: null,
dirty: false,
justDownloaded: false,

View file

@ -3,21 +3,13 @@ import SyncManager from "src/sync-manager";
import { usePlugin } from "src/views/hooks";
import { DEFAULT_SETTINGS, GitHubSyncSettings } from "src/settings/settings";
const STEPS = [
"welcome",
"repo",
"token",
"folders",
"sync",
"first_sync",
] as const;
const STEPS = ["welcome", "repo", "token", "sync", "first_sync"] as const;
type Step = (typeof STEPS)[number];
type StepData = {
repo: { owner: string; repo: string; branch: string };
token: { token: string };
folders: { repoFolder: string; vaultFolder: string };
sync: {
mode: "manual" | "interval";
syncOnStart: boolean;
@ -29,7 +21,6 @@ type StepData = {
const DEFAULT_STEP_DATA: StepData = {
repo: { owner: "", repo: "", branch: "" },
token: { token: "" },
folders: { repoFolder: "", vaultFolder: "" },
sync: {
mode: "manual",
syncOnStart: false,
@ -187,62 +178,6 @@ const TokenStepComponent = ({
);
};
const FoldersStepComponent = ({
values,
onChange,
}: {
values: { repoFolder: string; vaultFolder: string };
onChange: (values: { repoFolder: string; vaultFolder: string }) => void;
}) => {
return (
<div
style={{
display: "flex",
flexFlow: "column",
flexGrow: 1,
justifyContent: "center",
alignItems: "center",
}}
>
<p style={{ textAlign: "center" }}>
Optionally you can sync only part of your remote repository or your
vault.
</p>
<p style={{ textAlign: "center" }}>
Leave blank to sync the whole repository and the whole vault.
</p>
<p style={{ textAlign: "center" }}>
If you sync only a part of this vault remember you must always reuse the
same local path if you want to sync other vaults too.
</p>
<div
style={{
display: "flex",
flexFlow: "column",
gap: "var(--size-4-2)",
}}
>
<input
type="text"
spellCheck="false"
placeholder="Repository folder"
value={values.repoFolder}
onChange={(e) => onChange({ ...values, repoFolder: e.target.value })}
style={{ width: "100%" }}
/>
<input
type="text"
spellCheck="false"
placeholder="Vault folder"
value={values.vaultFolder}
onChange={(e) => onChange({ ...values, vaultFolder: e.target.value })}
style={{ width: "100%" }}
/>
</div>
</div>
);
};
const SyncSettingsStepComponent = ({
values,
onChange,
@ -356,8 +291,6 @@ const FirstSyncStepComponent = ({
githubOwner: stepData.repo.owner,
githubRepo: stepData.repo.repo,
githubBranch: stepData.repo.branch,
repoContentDir: stepData.folders.repoFolder,
localContentDir: stepData.folders.vaultFolder,
syncStrategy: stepData.sync.mode,
syncOnStartup: stepData.sync.syncOnStart,
syncConfigDir: stepData.sync.syncConfigDir,
@ -518,13 +451,6 @@ const OnBoardingComponent = ({ onClose }: { onClose: () => Promise<void> }) => {
onChange={(data) => updateStepData("token", data)}
/>
);
case "folders":
return (
<FoldersStepComponent
values={stepData.folders}
onChange={(data) => updateStepData("folders", data)}
/>
);
case "sync":
return (
<SyncSettingsStepComponent