Ditch partial sync

This commit is contained in:
Silvano Cerza 2025-01-23 11:09:58 +01:00
parent 923a811a4a
commit d20bda9a68
8 changed files with 63 additions and 272 deletions

View file

@ -12,8 +12,6 @@ These are the main features of the plugin:
- Desktop and mobile support - Desktop and mobile support
- Doesn't require `git` - Doesn't require `git`
- Multiple vaults sync - Multiple vaults sync
- Partial vault sync
- Partial remote repository sync
- Automatic sync on fixed interval - Automatic sync on fixed interval
- Manual sync - Manual sync
@ -63,15 +61,6 @@ If you don't want to see the button you can hide it, just check the plugin setti
The `Sync with GitHub` command is also available. The `Sync with GitHub` command is also available.
### Partial sync
Optionally you can choose to sync only a part of your vault and/or your repository.
This might be useful if you keep the sources for your blog in your vault as example, though you sync with some other service.
> [!WARNING]
> For the time being changing the directories after onboarding is not supported.
> I mean, you can do it, though I have not tested it and it's likely to break in unforeseeable ways.
### Config sync ### Config sync
If you want to sync your vault configs with other vault you can enable that. If you want to sync your vault configs with other vault you can enable that.

View file

@ -38,23 +38,8 @@ export default class EventsListener {
return; return;
} }
let remotePath: string;
if (file.path.startsWith(this.settings.localContentDir)) {
remotePath = file.path.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
} else if (
file.path.startsWith(`${this.vault.configDir}/github-sync-metadata.json`)
) {
remotePath = file.path;
} else {
throw new Error("Unexpected file path");
}
this.metadataStore.data.files[file.path] = { this.metadataStore.data.files[file.path] = {
localPath: file.path, path: file.path,
remotePath: remotePath!,
sha: null, sha: null,
dirty: true, dirty: true,
// This file has been created by the user // This file has been created by the user
@ -132,9 +117,12 @@ export default class EventsListener {
private isSyncable(filePath: string) { private isSyncable(filePath: string) {
return ( return (
filePath.startsWith(this.settings.localContentDir) ||
filePath === `${this.vault.configDir}/github-sync-metadata.json` || filePath === `${this.vault.configDir}/github-sync-metadata.json` ||
(this.settings.syncConfigDir && filePath.startsWith(this.vault.configDir)) (this.settings.syncConfigDir &&
filePath.startsWith(this.vault.configDir)) ||
// Obsidian recommends not syncing the workspace files
filePath === `${this.vault.configDir}/workspace.json` ||
filePath === `${this.vault.configDir}/workspace-mobile.json`
); );
} }
} }

View file

@ -43,8 +43,6 @@ export default class GithubClient {
private owner: string, private owner: string,
private repo: string, private repo: string,
private branch: string, private branch: string,
private repoContentDir: string,
private configDir: string,
) {} ) {}
headers() { headers() {

View file

@ -6,9 +6,7 @@ import { Vault, normalizePath } from "obsidian";
*/ */
export interface FileMetadata { export interface FileMetadata {
// Local path to the file // Local path to the file
localPath: string; path: string;
// Path to the file in the remote repository.
remotePath: string;
// SHA of the file in the remote repository. // SHA of the file in the remote repository.
// This is necessary to update the file remotely. // This is necessary to update the file remotely.
// If this is null the file has not yet been pushed to the remote repository. // If this is null the file has not yet been pushed to the remote repository.

View file

@ -4,8 +4,6 @@ export interface GitHubSyncSettings {
githubOwner: string; githubOwner: string;
githubRepo: string; githubRepo: string;
githubBranch: string; githubBranch: string;
repoContentDir: string;
localContentDir: string;
syncStrategy: "manual" | "interval"; syncStrategy: "manual" | "interval";
syncInterval: number; syncInterval: number;
syncOnStartup: boolean; syncOnStartup: boolean;
@ -21,8 +19,6 @@ export const DEFAULT_SETTINGS: GitHubSyncSettings = {
githubOwner: "", githubOwner: "",
githubRepo: "", githubRepo: "",
githubBranch: "main", githubBranch: "main",
repoContentDir: "",
localContentDir: "",
syncStrategy: "manual", syncStrategy: "manual",
syncInterval: 1, syncInterval: 1,
syncOnStartup: false, syncOnStartup: false,

View file

@ -85,42 +85,6 @@ export default class GitHubSyncSettingsTab extends PluginSettingTab {
}), }),
); );
containerEl.createEl("h2", { text: "Folders" });
new Setting(containerEl)
.setName("Repository content directory")
.setDesc(
`The repository directory to sync, relative to the repository root.
If not set the whole repository will be synced.`,
)
.addText((text) =>
text
.setPlaceholder("Exaple: blog/content")
.setValue(this.plugin.settings.repoContentDir)
.onChange(async (value) => {
// TODO: Change the local path if already fetched
this.plugin.settings.repoContentDir = value;
await this.plugin.saveSettings();
}),
);
new Setting(containerEl)
.setName("Local content directory")
.setDesc(
`The local directory to sync, relative to the vault root.
If not set the whole vault will be synced.`,
)
.addText((text) =>
text
.setPlaceholder("Exaple: folder/blog-posts")
.setValue(this.plugin.settings.localContentDir)
.onChange(async (value) => {
// TODO: Move the folder if already fetched
this.plugin.settings.localContentDir = value;
await this.plugin.saveSettings();
}),
);
containerEl.createEl("h2", { text: "Sync" }); containerEl.createEl("h2", { text: "Sync" });
const syncStrategies = { const syncStrategies = {

View file

@ -34,8 +34,6 @@ export default class SyncManager {
this.settings.githubOwner, this.settings.githubOwner,
this.settings.githubRepo, this.settings.githubRepo,
this.settings.githubBranch, this.settings.githubBranch,
this.settings.repoContentDir,
this.vault.configDir,
); );
this.eventsListener = new EventsListener( this.eventsListener = new EventsListener(
this.vault, this.vault,
@ -45,45 +43,23 @@ export default class SyncManager {
} }
/** /**
* Returns true if the remote content dir is empty. * Returns true if the local vault root is empty.
* @param files All files in the remote repository
*/ */
private remoteContentDirIsEmpty(files: { private async vaultIsEmpty(): Promise<boolean> {
[key: string]: GetTreeResponseItem; const { files, folders } = await this.vault.adapter.list(
}): boolean { this.vault.getRoot().path,
);
// There are files or folders in the vault dir
return ( return (
Object.keys(files).filter((filePath: string) => files.length === 0 ||
filePath.startsWith(this.settings.repoContentDir), // We filter out the config dir since is always present so it's fine if we find it.
).length === 0 folders.filter((f) => f !== this.vault.configDir).length === 0
); );
} }
/**
* Returns true if the local content dir is empty.
* If the local content dir is the vault root the config dir is ignored.
*/
private async localContentDirIsEmpty(): Promise<boolean> {
const localContentDirExists = await this.vault.adapter.exists(
this.settings.localContentDir,
);
if (localContentDirExists) {
const { files, folders } = await this.vault.adapter.list(
this.settings.localContentDir,
);
// There are files or folders in the local content dir
return (
files.length === 0 ||
// We filter out the config dir in case the user wants to sync the whole
// vault. The config dir is always present so it's fine if we find it.
folders.filter((f) => f !== this.vault.configDir).length === 0
);
}
return true;
}
/** /**
* Handles first sync with remote and local. * Handles first sync with remote and local.
* This fails neither remote nor local folders are empty. * This fails if neither remote nor local folders are empty.
*/ */
async firstSync() { async firstSync() {
let repositoryIsBare = false; let repositoryIsBare = false;
@ -122,21 +98,21 @@ export default class SyncManager {
treeSha = res.sha; treeSha = res.sha;
} }
const remoteContentDirIsEmpty = this.remoteContentDirIsEmpty(files); const remoteRepoIsEmpty = Object.keys(files).length === 0;
const localContentDirIsEmpty = await this.localContentDirIsEmpty(); const vaultIsEmpty = await this.vaultIsEmpty();
if (!remoteContentDirIsEmpty && !localContentDirIsEmpty) { if (!remoteRepoIsEmpty && !vaultIsEmpty) {
// Both have files, we can't sync, show error // Both have files, we can't sync, show error
throw new Error("Both remote and local have files, can't sync"); throw new Error("Both remote and local have files, can't sync");
} else if (remoteContentDirIsEmpty || repositoryIsBare) { } else if (remoteRepoIsEmpty || repositoryIsBare) {
// Remote has no files and no manifest, let's just upload whatever we have locally. // Remote has no files and no manifest, let's just upload whatever we have locally.
// This is fine even if the local content dir is empty. // This is fine even if the vault is empty.
// The most important thing at this point is that the remote manifest is created. // The most important thing at this point is that the remote manifest is created.
await this.firstSyncFromLocal(files, treeSha); await this.firstSyncFromLocal(files, treeSha);
} else { } else {
// Local has no files and there's no manifest in the remote repo. // Local has no files and there's no manifest in the remote repo.
// Let's download whatever we have in the remote content dir. // Let's download whatever we have in the remote repo.
// This is fine even if the remote content dir is empty. // This is fine even if the remote repo is empty.
// In this case too the important step is that the remote manifest is created. // In this case too the important step is that the remote manifest is created.
await this.firstSyncFromRemote(files, treeSha); await this.firstSyncFromRemote(files, treeSha);
} }
@ -157,21 +133,16 @@ export default class SyncManager {
await Promise.all( await Promise.all(
Object.keys(files) Object.keys(files)
.filter((filePath: string) => { .filter((filePath: string) => {
if (filePath.startsWith(this.settings.repoContentDir)) { if (
return true;
} else if (
filePath === `${this.vault.configDir}/github-sync-metadata.json`
) {
// The metadata file must always be synced
return true;
} else if (
this.settings.syncConfigDir && this.settings.syncConfigDir &&
filePath.startsWith(this.vault.configDir) filePath.startsWith(this.vault.configDir) &&
filePath !== `${this.vault.configDir}/github-sync-metadata.json`
) { ) {
// Include files in the config dir only if the user has enabled it // Include files in the config dir only if the user has enabled it.
return true; // The metadata file must always be synced.
return false;
} }
return false; return true;
}) })
.map(async (filePath: string) => { .map(async (filePath: string) => {
await this.downloadFile(files[filePath], Date.now()); await this.downloadFile(files[filePath], Date.now());
@ -197,9 +168,8 @@ export default class SyncManager {
async (filePath: string) => { async (filePath: string) => {
const normalizedPath = normalizePath(filePath); const normalizedPath = normalizePath(filePath);
const content = await this.vault.adapter.read(normalizedPath); const content = await this.vault.adapter.read(normalizedPath);
const { remotePath } = this.metadataStore.data.files[filePath]; newTreeFiles[filePath] = {
newTreeFiles[remotePath] = { path: filePath,
path: remotePath,
mode: "100644", mode: "100644",
type: "blob", type: "blob",
content: content, content: content,
@ -212,10 +182,10 @@ export default class SyncManager {
/** /**
* Handles first sync with the remote repository. * Handles first sync with the remote repository.
* This must be called in case there are no files in the remote content dir and no manifest while * This must be called in case there are no files in the remote repo and no manifest while
* local content dir has files and a manifest. * local vault has files and a manifest.
* *
* @param files All files in the remote repository, including those not in its content dir. * @param files All files in the remote repository
* @param treeSha The SHA of the tree in the remote repository. * @param treeSha The SHA of the tree in the remote repository.
*/ */
async firstSyncFromLocal( async firstSyncFromLocal(
@ -241,9 +211,8 @@ export default class SyncManager {
async (filePath: string) => { async (filePath: string) => {
const normalizedPath = normalizePath(filePath); const normalizedPath = normalizePath(filePath);
const content = await this.vault.adapter.read(normalizedPath); const content = await this.vault.adapter.read(normalizedPath);
const { remotePath } = this.metadataStore.data.files[filePath]; newTreeFiles[filePath] = {
newTreeFiles[remotePath] = { path: filePath,
path: remotePath,
mode: "100644", mode: "100644",
type: "blob", type: "blob",
content: content, content: content,
@ -286,12 +255,12 @@ export default class SyncManager {
if (resolution) { if (resolution) {
conflictResolutions.push({ conflictResolutions.push({
type: "download", type: "download",
filePath: conflicts[index].remoteFile.localPath, filePath: conflicts[index].remoteFile.path,
}); });
} else { } else {
conflictResolutions.push({ conflictResolutions.push({
type: "upload", type: "upload",
filePath: conflicts[index].localFile.localPath, filePath: conflicts[index].localFile.path,
}); });
} }
}, },
@ -334,12 +303,8 @@ export default class SyncManager {
case "upload": { case "upload": {
const normalizedPath = normalizePath(action.filePath); const normalizedPath = normalizePath(action.filePath);
const content = await this.vault.adapter.read(normalizedPath); const content = await this.vault.adapter.read(normalizedPath);
const remotePath = action.filePath.replace( newTreeFiles[action.filePath] = {
this.settings.localContentDir, path: action.filePath,
this.settings.repoContentDir,
);
newTreeFiles[remotePath] = {
path: remotePath,
mode: "100644", mode: "100644",
type: "blob", type: "blob",
content: content, content: content,
@ -347,9 +312,7 @@ export default class SyncManager {
break; break;
} }
case "delete_remote": { case "delete_remote": {
const { remotePath } = newTreeFiles[action.filePath].sha = null;
this.metadataStore.data.files[action.filePath];
newTreeFiles[remotePath].sha = null;
break; break;
} }
case "download": case "download":
@ -365,12 +328,8 @@ export default class SyncManager {
...actions ...actions
.filter((action) => action.type === "download") .filter((action) => action.type === "download")
.map(async (action: SyncAction) => { .map(async (action: SyncAction) => {
const remotePath = action.filePath.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
await this.downloadFile( await this.downloadFile(
files[remotePath], files[action.filePath],
remoteMetadata.files[action.filePath].lastModified, remoteMetadata.files[action.filePath].lastModified,
); );
}), }),
@ -386,8 +345,8 @@ export default class SyncManager {
/** /**
* Finds conflicts between local and remote files. * Finds conflicts between local and remote files.
* @param remoteFiles All files in the remote content dir * @param remoteFiles All files in the remote repo
* @param localFiles All files in the local content dir * @param localFiles All files in the local vault
* @returns List of objects with both remote and local conflicting files metadata * @returns List of objects with both remote and local conflicting files metadata
*/ */
findConflicts( findConflicts(
@ -432,8 +391,8 @@ export default class SyncManager {
/** /**
* Determines which sync action to take for each file. * Determines which sync action to take for each file.
* *
* @param remoteFiles All files in the remote content dir * @param remoteFiles All files in the remote repo
* @param localFiles All files in the local content dir * @param localFiles All files in the local vault
* *
* @returns List of SyncActions * @returns List of SyncActions
*/ */
@ -589,30 +548,26 @@ export default class SyncManager {
async downloadFile(file: GetTreeResponseItem, lastModified: number) { async downloadFile(file: GetTreeResponseItem, lastModified: number) {
const url = file.url; const url = file.url;
const destinationFile = file.path.replace( const fileMetadata = this.metadataStore.data.files[file.path];
this.settings.repoContentDir,
this.settings.localContentDir,
);
const fileMetadata = this.metadataStore.data.files[destinationFile];
if (fileMetadata && fileMetadata.sha === file.sha) { if (fileMetadata && fileMetadata.sha === file.sha) {
// File already exists and has the same SHA, no need to download it again. // File already exists and has the same SHA, no need to download it again.
return; return;
} }
const blob = await this.client.getBlob(url); const blob = await this.client.getBlob(url);
const destinationFolder = normalizePath( const normalizedPath = normalizePath(file.path);
destinationFile.split("/").slice(0, -1).join("/"), const fileFolder = normalizePath(
normalizedPath.split("/").slice(0, -1).join("/"),
); );
if (!(await this.vault.adapter.exists(destinationFolder))) { if (!(await this.vault.adapter.exists(fileFolder))) {
await this.vault.adapter.mkdir(destinationFolder); await this.vault.adapter.mkdir(fileFolder);
} }
this.vault.adapter.writeBinary( this.vault.adapter.writeBinary(
normalizePath(destinationFile), normalizedPath,
Buffer.from(blob.content, "base64"), Buffer.from(blob.content, "base64"),
); );
this.metadataStore.data.files[destinationFile] = { this.metadataStore.data.files[file.path] = {
localPath: destinationFile, path: file.path,
remotePath: file.path,
sha: file.sha, sha: file.sha,
dirty: false, dirty: false,
justDownloaded: true, justDownloaded: true,
@ -633,7 +588,7 @@ export default class SyncManager {
await this.metadataStore.load(); await this.metadataStore.load();
if (Object.keys(this.metadataStore.data.files).length === 0) { if (Object.keys(this.metadataStore.data.files).length === 0) {
let files = []; let files = [];
let folders = [this.settings.localContentDir]; let folders = [this.vault.getRoot().path];
while (folders.length > 0) { while (folders.length > 0) {
const folder = folders.pop(); const folder = folders.pop();
if (folder === undefined) { if (folder === undefined) {
@ -653,28 +608,8 @@ export default class SyncManager {
return; return;
} }
// We change the remote path only if the file is not in the config dir
// as that directory is always at the root of the repo.
// If it's synced obviously.
let remotePath = filePath;
if (!filePath.startsWith(this.vault.configDir)) {
if (this.settings.localContentDir === "") {
if (this.settings.repoContentDir === "") {
remotePath = filePath;
} else {
remotePath = `${this.settings.repoContentDir}/${filePath}`;
}
} else {
remotePath = filePath.replace(
this.settings.localContentDir,
this.settings.repoContentDir,
);
}
}
this.metadataStore.data.files[filePath] = { this.metadataStore.data.files[filePath] = {
localPath: filePath, path: filePath,
remotePath: remotePath,
sha: null, sha: null,
dirty: false, dirty: false,
justDownloaded: false, justDownloaded: false,
@ -683,12 +618,11 @@ export default class SyncManager {
}); });
// Must be the first time we run, initialize the metadata store // Must be the first time we run, initialize the metadata store
// with itself and all files in the local content dir. // with itself and all files in the vault.
this.metadataStore.data.files[ this.metadataStore.data.files[
`${this.vault.configDir}/github-sync-metadata.json` `${this.vault.configDir}/github-sync-metadata.json`
] = { ] = {
localPath: `${this.vault.configDir}/github-sync-metadata.json`, path: `${this.vault.configDir}/github-sync-metadata.json`,
remotePath: `${this.vault.configDir}/github-sync-metadata.json`,
sha: null, sha: null,
dirty: false, dirty: false,
justDownloaded: false, justDownloaded: false,
@ -719,9 +653,7 @@ export default class SyncManager {
// Add them to the metadata store // Add them to the metadata store
files.forEach((filePath: string) => { files.forEach((filePath: string) => {
this.metadataStore.data.files[filePath] = { this.metadataStore.data.files[filePath] = {
localPath: filePath, path: filePath,
// Remote path is the same for config dir files
remotePath: filePath,
sha: null, sha: null,
dirty: false, dirty: false,
justDownloaded: false, justDownloaded: false,

View file

@ -3,21 +3,13 @@ import SyncManager from "src/sync-manager";
import { usePlugin } from "src/views/hooks"; import { usePlugin } from "src/views/hooks";
import { DEFAULT_SETTINGS, GitHubSyncSettings } from "src/settings/settings"; import { DEFAULT_SETTINGS, GitHubSyncSettings } from "src/settings/settings";
const STEPS = [ const STEPS = ["welcome", "repo", "token", "sync", "first_sync"] as const;
"welcome",
"repo",
"token",
"folders",
"sync",
"first_sync",
] as const;
type Step = (typeof STEPS)[number]; type Step = (typeof STEPS)[number];
type StepData = { type StepData = {
repo: { owner: string; repo: string; branch: string }; repo: { owner: string; repo: string; branch: string };
token: { token: string }; token: { token: string };
folders: { repoFolder: string; vaultFolder: string };
sync: { sync: {
mode: "manual" | "interval"; mode: "manual" | "interval";
syncOnStart: boolean; syncOnStart: boolean;
@ -29,7 +21,6 @@ type StepData = {
const DEFAULT_STEP_DATA: StepData = { const DEFAULT_STEP_DATA: StepData = {
repo: { owner: "", repo: "", branch: "" }, repo: { owner: "", repo: "", branch: "" },
token: { token: "" }, token: { token: "" },
folders: { repoFolder: "", vaultFolder: "" },
sync: { sync: {
mode: "manual", mode: "manual",
syncOnStart: false, syncOnStart: false,
@ -187,62 +178,6 @@ const TokenStepComponent = ({
); );
}; };
const FoldersStepComponent = ({
values,
onChange,
}: {
values: { repoFolder: string; vaultFolder: string };
onChange: (values: { repoFolder: string; vaultFolder: string }) => void;
}) => {
return (
<div
style={{
display: "flex",
flexFlow: "column",
flexGrow: 1,
justifyContent: "center",
alignItems: "center",
}}
>
<p style={{ textAlign: "center" }}>
Optionally you can sync only part of your remote repository or your
vault.
</p>
<p style={{ textAlign: "center" }}>
Leave blank to sync the whole repository and the whole vault.
</p>
<p style={{ textAlign: "center" }}>
If you sync only a part of this vault remember you must always reuse the
same local path if you want to sync other vaults too.
</p>
<div
style={{
display: "flex",
flexFlow: "column",
gap: "var(--size-4-2)",
}}
>
<input
type="text"
spellCheck="false"
placeholder="Repository folder"
value={values.repoFolder}
onChange={(e) => onChange({ ...values, repoFolder: e.target.value })}
style={{ width: "100%" }}
/>
<input
type="text"
spellCheck="false"
placeholder="Vault folder"
value={values.vaultFolder}
onChange={(e) => onChange({ ...values, vaultFolder: e.target.value })}
style={{ width: "100%" }}
/>
</div>
</div>
);
};
const SyncSettingsStepComponent = ({ const SyncSettingsStepComponent = ({
values, values,
onChange, onChange,
@ -356,8 +291,6 @@ const FirstSyncStepComponent = ({
githubOwner: stepData.repo.owner, githubOwner: stepData.repo.owner,
githubRepo: stepData.repo.repo, githubRepo: stepData.repo.repo,
githubBranch: stepData.repo.branch, githubBranch: stepData.repo.branch,
repoContentDir: stepData.folders.repoFolder,
localContentDir: stepData.folders.vaultFolder,
syncStrategy: stepData.sync.mode, syncStrategy: stepData.sync.mode,
syncOnStartup: stepData.sync.syncOnStart, syncOnStartup: stepData.sync.syncOnStart,
syncConfigDir: stepData.sync.syncConfigDir, syncConfigDir: stepData.sync.syncConfigDir,
@ -518,13 +451,6 @@ const OnBoardingComponent = ({ onClose }: { onClose: () => Promise<void> }) => {
onChange={(data) => updateStepData("token", data)} onChange={(data) => updateStepData("token", data)}
/> />
); );
case "folders":
return (
<FoldersStepComponent
values={stepData.folders}
onChange={(data) => updateStepData("folders", data)}
/>
);
case "sync": case "sync":
return ( return (
<SyncSettingsStepComponent <SyncSettingsStepComponent