silvanocerza_github-gitless-sync/src/sync-manager.ts

import { Vault, normalizePath } from "obsidian";
import GithubClient, {
  GetTreeResponseItem,
  NewTreeRequestItem,
  RepoContent,
} from "./github/client";
import MetadataStore, { FileMetadata, Metadata } from "./metadata-store";
import EventsListener from "./events-listener";
import { GitHubSyncSettings } from "./settings/settings";

interface SyncAction {
  type: "upload" | "download" | "delete_local" | "delete_remote";
  filePath: string;
}

type OnConflictsCallback = (
  conflicts: { remoteFile: FileMetadata; localFile: FileMetadata }[],
) => Promise<boolean[]>;

export default class SyncManager {
  private metadataStore: MetadataStore;
  private client: GithubClient;
  private eventsListener: EventsListener;
  private syncIntervalId: number | null = null;

  constructor(
    private vault: Vault,
    private settings: GitHubSyncSettings,
    private onConflicts: OnConflictsCallback,
  ) {
    this.metadataStore = new MetadataStore(this.vault);
    this.client = new GithubClient(
      this.settings.githubToken,
      this.settings.githubOwner,
      this.settings.githubRepo,
      this.settings.githubBranch,
      this.settings.repoContentDir,
      this.vault.configDir,
    );
    this.eventsListener = new EventsListener(
      this.vault,
      this.metadataStore,
      this.settings,
    );
  }

  /**
   * Returns true if the remote content dir is empty.
   * @param files All files in the remote repository
   */
  private remoteContentDirIsEmpty(files: {
    [key: string]: GetTreeResponseItem;
  }): boolean {
    return (
      Object.keys(files).filter((filePath: string) =>
        filePath.startsWith(this.settings.repoContentDir),
      ).length === 0
    );
  }

  /**
   * Returns true if the local content dir is empty.
   * If the local content dir is the vault root the config dir is ignored.
   */
  private async localContentDirIsEmpty(): Promise<boolean> {
    const localContentDirExists = await this.vault.adapter.exists(
      this.settings.localContentDir,
    );
    if (localContentDirExists) {
      const { files, folders } = await this.vault.adapter.list(
        this.settings.localContentDir,
      );
      // There are files or folders in the local content dir
      return (
        files.length === 0 ||
        // We filter out the config dir in case the user wants to sync the whole
        // vault. The config dir is always present so it's fine if we find it.
        folders.filter((f) => f !== this.vault.configDir).length === 0
      );
    }
    return true;
  }

  /**
   * Handles first sync with remote and local.
   * This fails neither remote nor local folders are empty.
   */
  async firstSync() {
    let repositoryIsBare = false;
    let res: RepoContent;
    let files: {
      [key: string]: GetTreeResponseItem;
    } = {};
    let treeSha: string = "";
    try {
      res = await this.client.getRepoContent();
      files = res.files;
      treeSha = res.sha;
    } catch (err) {
      if (err.status !== 409) {
        throw err;
      }
      // The repository is bare, meaning it has no tree, no commits and no branches
      repositoryIsBare = true;
    }

    if (repositoryIsBare) {
      // Since the repository is completely empty we need to create a first commit.
      // We can't create that by going throught the normal sync process since the
      // API doesn't let us create a new tree when the repo is empty.
      // So we create a the manifest file as the first commit, since we're going
      // to create that in any case right after this.
      await this.client.createFile(
        `${this.vault.configDir}/github-sync-metadata.json`,
        "",
        "Initial commit",
      );
      // Now get the repo content again cause we know for sure it will return a
      // valid sha that we can use to create the first sync commit.
      res = await this.client.getRepoContent();
      files = res.files;
      treeSha = res.sha;
    }

    const remoteContentDirIsEmpty = this.remoteContentDirIsEmpty(files);
    const localContentDirIsEmpty = await this.localContentDirIsEmpty();

    if (!remoteContentDirIsEmpty && !localContentDirIsEmpty) {
      // Both have files, we can't sync, show error
      throw new Error("Both remote and local have files, can't sync");
    } else if (remoteContentDirIsEmpty || repositoryIsBare) {
      // Remote has no files and no manifest, let's just upload whatever we have locally.
      // This is fine even if the local content dir is empty.
      // The most important thing at this point is that the remote manifest is created.
      await this.firstSyncFromLocal(files, treeSha);
    } else {
      // Local has no files and there's no manifest in the remote repo.
      // Let's download whatever we have in the remote content dir.
      // This is fine even if the remote content dir is empty.
      // In this case too the important step is that the remote manifest is created.
      await this.firstSyncFromRemote(files, treeSha);
    }
  }

  /**
   * Handles first sync with the remote repository.
   * This must be called in case there are no files in the local content dir while
   * remote has files in the repo content dir but no manifest file.
   *
   * @param files All files in the remote repository, including those not in its content dir.
   * @param treeSha The SHA of the tree in the remote repository.
   */
  async firstSyncFromRemote(
    files: { [key: string]: GetTreeResponseItem },
    treeSha: string,
  ) {
    await Promise.all(
      Object.keys(files)
        .filter((filePath: string) => {
          if (filePath.startsWith(this.settings.repoContentDir)) {
            return true;
          } else if (
            filePath === `${this.vault.configDir}/github-sync-metadata.json`
          ) {
            // The metadata file must always be synced
            return true;
          } else if (
            this.settings.syncConfigDir &&
            filePath.startsWith(this.vault.configDir)
          ) {
            // Include files in the config dir only if the user has enabled it
            return true;
          }
          return false;
        })
        .map(async (filePath: string) => {
          await this.downloadFile(files[filePath], Date.now());
        }),
    );
    const newTreeFiles = Object.keys(files)
      .map((filePath: string) => ({
        path: files[filePath].path,
        mode: files[filePath].mode,
        type: files[filePath].type,
        sha: files[filePath].sha,
      }))
      .reduce(
        (
          acc: { [key: string]: NewTreeRequestItem },
          item: NewTreeRequestItem,
        ) => ({ ...acc, [item.path]: item }),
        {},
      );
    // Add files that are in the manifest but not in the tree.
    await Promise.all(
      Object.keys(this.metadataStore.data.files).map(
        async (filePath: string) => {
          const normalizedPath = normalizePath(filePath);
          const content = await this.vault.adapter.read(normalizedPath);
          const { remotePath } = this.metadataStore.data.files[filePath];
          newTreeFiles[remotePath] = {
            path: remotePath,
            mode: "100644",
            type: "blob",
            content: content,
          };
        },
      ),
    );
    await this.commitSync(newTreeFiles, treeSha);
  }

  /**
   * Handles first sync with the remote repository.
   * This must be called in case there are no files in the remote content dir and no manifest while
   * local content dir has files and a manifest.
   *
   * @param files All files in the remote repository, including those not in its content dir.
   * @param treeSha The SHA of the tree in the remote repository.
   */
  async firstSyncFromLocal(
    files: { [key: string]: GetTreeResponseItem },
    treeSha: string,
  ) {
    const newTreeFiles = Object.keys(files)
      .map((filePath: string) => ({
        path: files[filePath].path,
        mode: files[filePath].mode,
        type: files[filePath].type,
        sha: files[filePath].sha,
      }))
      .reduce(
        (
          acc: { [key: string]: NewTreeRequestItem },
          item: NewTreeRequestItem,
        ) => ({ ...acc, [item.path]: item }),
        {},
      );
    await Promise.all(
      Object.keys(this.metadataStore.data.files).map(
        async (filePath: string) => {
          const normalizedPath = normalizePath(filePath);
          const content = await this.vault.adapter.read(normalizedPath);
          const { remotePath } = this.metadataStore.data.files[filePath];
          newTreeFiles[remotePath] = {
            path: remotePath,
            mode: "100644",
            type: "blob",
            content: content,
          };
        },
      ),
    );
    await this.commitSync(newTreeFiles, treeSha);
  }

  /**
   * Syncs local and remote folders.
   * @returns
   */
  async sync() {
    const { files, sha: treeSha } = await this.client.getRepoContent();
    const manifest = files[`${this.vault.configDir}/github-sync-metadata.json`];

    if (manifest === undefined) {
      throw new Error("Remote manifest is missing");
    }

    const blob = await this.client.getBlob(manifest.url);
    const remoteMetadata: Metadata = JSON.parse(atob(blob.content));

    const conflicts = this.findConflicts(
      remoteMetadata.files,
      this.metadataStore.data.files,
    );
    let conflictResolutions: SyncAction[] = [];
    if (conflicts.length > 0) {
      // TODO: Show conflicts to the user with a callback
      // Wait for response
      // Create new action to handle the conflict
      // Add the new action to the list later on
      console.log("Conflicts");
      console.log(conflicts);
      (await this.onConflicts(conflicts)).forEach(
        (resolution: boolean, index: number) => {
          if (resolution) {
            conflictResolutions.push({
              type: "download",
              filePath: conflicts[index].remoteFile.localPath,
            });
          } else {
            conflictResolutions.push({
              type: "upload",
              filePath: conflicts[index].localFile.localPath,
            });
          }
        },
      );
    }

    const actions = [
      ...this.determineSyncActions(
        remoteMetadata.files,
        this.metadataStore.data.files,
      ),
      ...conflictResolutions,
    ];

    if (actions.length === 0) {
      // Nothing to sync
      return;
    }

    const newTreeFiles: { [key: string]: NewTreeRequestItem } = Object.keys(
      files,
    )
      .map((filePath: string) => ({
        path: files[filePath].path,
        mode: files[filePath].mode,
        type: files[filePath].type,
        sha: files[filePath].sha,
      }))
      .reduce(
        (
          acc: { [key: string]: NewTreeRequestItem },
          item: NewTreeRequestItem,
        ) => ({ ...acc, [item.path]: item }),
        {},
      );

    await Promise.all(
      actions.map(async (action) => {
        switch (action.type) {
          case "upload": {
            const normalizedPath = normalizePath(action.filePath);
            const content = await this.vault.adapter.read(normalizedPath);
            const remotePath = action.filePath.replace(
              this.settings.localContentDir,
              this.settings.repoContentDir,
            );
            newTreeFiles[remotePath] = {
              path: remotePath,
              mode: "100644",
              type: "blob",
              content: content,
            };
            break;
          }
          case "delete_remote": {
            const { remotePath } =
              this.metadataStore.data.files[action.filePath];
            newTreeFiles[remotePath].sha = null;
            break;
          }
          case "download":
            break;
          case "delete_local":
            break;
        }
      }),
    );

    // Download files and delete local files
    await Promise.all([
      ...actions
        .filter((action) => action.type === "download")
        .map(async (action: SyncAction) => {
          const remotePath = action.filePath.replace(
            this.settings.localContentDir,
            this.settings.repoContentDir,
          );
          await this.downloadFile(
            files[remotePath],
            remoteMetadata.files[action.filePath].lastModified,
          );
        }),
      ...actions
        .filter((action) => action.type === "delete_local")
        .map(async (action: SyncAction) => {
          await this.deleteLocalFile(action.filePath);
        }),
    ]);

    await this.commitSync(newTreeFiles, treeSha);
  }

  /**
   * Finds conflicts between local and remote files.
   * @param remoteFiles All files in the remote content dir
   * @param localFiles All files in the local content dir
   * @returns List of objects with both remote and local conflicting files metadata
   */
  findConflicts(
    remoteFiles: { [key: string]: FileMetadata },
    localFiles: { [key: string]: FileMetadata },
  ): { remoteFile: FileMetadata; localFile: FileMetadata }[] {
    const commonFiles = Object.keys(remoteFiles).filter(
      (key) => key in localFiles,
    );

    return commonFiles
      .map((filePath: string) => {
        const remoteFile = remoteFiles[filePath];
        const localFile = localFiles[filePath];

        // We compare the SHA cause it remote files changes the SHA changes
        // but not the same happens when the file is modified locally.
        // So if sha are different and the local file is newer we can't
        // know for sure which version should be kept.
        if (
          remoteFile.sha !== localFile.sha &&
          remoteFile.lastModified < localFile.lastModified
        ) {
          // File is modified on both sides, the user must solve the conflict
          return {
            remoteFile: remoteFile,
            localFile: localFile,
          };
        }
        return null;
      })
      .filter(
        (
          conflict: {
            remoteFile: FileMetadata;
            localFile: FileMetadata;
          } | null,
        ) => conflict !== null,
      );
  }

  /**
   * Determines which sync action to take for each file.
   *
   * @param remoteFiles All files in the remote content dir
   * @param localFiles All files in the local content dir
   *
   * @returns List of SyncActions
   */
  determineSyncActions(
    remoteFiles: { [key: string]: FileMetadata },
    localFiles: { [key: string]: FileMetadata },
  ) {
    let actions: SyncAction[] = [];

    const commonFiles = Object.keys(remoteFiles).filter(
      (key) => key in localFiles,
    );

    // Get diff for common files
    commonFiles.forEach((filePath: string) => {
      const remoteFile = remoteFiles[filePath];
      const localFile = localFiles[filePath];
      if (remoteFile.deleted && localFile.deleted) {
        // Nothing to do
        return;
      }

      if (
        remoteFile.sha !== localFile.sha &&
        remoteFile.lastModified < localFile.lastModified
      ) {
        // This is a conflict, we handle it separately
        // We compare the SHA cause it remote files changes the SHA changes
        // but not the same happens when the file is modified locally.
        // So if sha are different and the local file is newer we can't
        // know for sure which version should be kept.
        return;
      }

      if (remoteFile.deleted && !localFile.deleted) {
        if ((remoteFile.deletedAt as number) > localFile.lastModified) {
          actions.push({
            type: "delete_local",
            filePath: filePath,
          });
        } else if (localFile.lastModified > (remoteFile.deletedAt as number)) {
          actions.push({ type: "upload", filePath: filePath });
        }
      }

      if (!remoteFile.deleted && localFile.deleted) {
        if (remoteFile.lastModified > (localFile.deletedAt as number)) {
          actions.push({ type: "download", filePath: filePath });
        } else if ((localFile.deletedAt as number) > remoteFile.lastModified) {
          actions.push({
            type: "delete_remote",
            filePath: filePath,
          });
        }
      }

      if (remoteFile.lastModified > localFile.lastModified) {
        actions.push({ type: "download", filePath: filePath });
      } else if (localFile.lastModified > remoteFile.lastModified) {
        actions.push({ type: "upload", filePath: filePath });
      }
    });

    // Get diff for files in remote but not in local
    Object.keys(remoteFiles).forEach((filePath: string) => {
      const remoteFile = remoteFiles[filePath];
      const localFile = localFiles[filePath];
      if (localFile) {
        // Local file exists, we already handled it.
        // Skip it.
        return;
      }
      if (remoteFile.deleted) {
        // Remote is deleted but we don't have it locally.
        // Nothing to do.
        // TODO: Maybe we need to remove remote reference too?
      } else {
        actions.push({ type: "download", filePath: filePath });
      }
    });

    // Get diff for files in local but not in remote
    Object.keys(localFiles).forEach((filePath: string) => {
      const remoteFile = remoteFiles[filePath];
      const localFile = localFiles[filePath];
      if (remoteFile) {
        // Remote file exists, we already handled it.
        // Skip it.
        return;
      }
      if (localFile.deleted) {
        // Local is deleted and remote doesn't exist.
        // Just remove the local reference.
      } else {
        actions.push({ type: "upload", filePath: filePath });
      }
    });

    if (!this.settings.syncConfigDir) {
      // Remove all actions that involve the config directory if the user doesn't want to sync it.
      // The manifest file is always synced.
      return actions.filter((action: SyncAction) => {
        return (
          !action.filePath.startsWith(this.vault.configDir) ||
          action.filePath ===
            `${this.vault.configDir}/github-sync-metadata.json`
        );
      });
    }

    return actions;
  }

  /**
   * Creates a new sync commit in the remote repository.
   *
   * @param treeFiles Updated list of files in the remote tree
   * @param baseTreeSha sha of the tree to use as base for the new tree
   */
  async commitSync(
    treeFiles: { [key: string]: NewTreeRequestItem },
    baseTreeSha: string,
  ) {
    // Update local sync time
    this.metadataStore.data.lastSync = Date.now();
    this.metadataStore.save();

    // Update manifest in list of new tree items
    delete treeFiles[`${this.vault.configDir}/github-sync-metadata.json`].sha;
    treeFiles[`${this.vault.configDir}/github-sync-metadata.json`].content =
      JSON.stringify(this.metadataStore.data);

    // Create the new tree
    const newTree: { tree: NewTreeRequestItem[]; base_tree: string } = {
      tree: Object.keys(treeFiles).map(
        (filePath: string) => treeFiles[filePath],
      ),
      base_tree: baseTreeSha,
    };
    const newTreeSha = await this.client.createTree(newTree);

    const branchHeadSha = await this.client.getBranchHeadSha();

    const commitSha = await this.client.createCommit(
      // TODO: Make this configurable or find a nicer commit message
      "Sync",
      newTreeSha,
      branchHeadSha,
    );

    await this.client.updateBranchHead(commitSha);
  }

  async downloadFile(file: GetTreeResponseItem, lastModified: number) {
    const url = file.url;
    const destinationFile = file.path.replace(
      this.settings.repoContentDir,
      this.settings.localContentDir,
    );
    const fileMetadata = this.metadataStore.data.files[destinationFile];
    if (fileMetadata && fileMetadata.sha === file.sha) {
      // File already exists and has the same SHA, no need to download it again.
      return;
    }

    const blob = await this.client.getBlob(url);
    const destinationFolder = normalizePath(
      destinationFile.split("/").slice(0, -1).join("/"),
    );
    if (!(await this.vault.adapter.exists(destinationFolder))) {
      await this.vault.adapter.mkdir(destinationFolder);
    }
    this.vault.adapter.writeBinary(
      normalizePath(destinationFile),
      Buffer.from(blob.content, "base64"),
    );
    this.metadataStore.data.files[destinationFile] = {
      localPath: destinationFile,
      remotePath: file.path,
      sha: file.sha,
      dirty: false,
      justDownloaded: true,
      lastModified: lastModified,
    };
    await this.metadataStore.save();
  }

  async deleteLocalFile(filePath: string) {
    const normalizedPath = normalizePath(filePath);
    await this.vault.adapter.remove(normalizedPath);
    this.metadataStore.data.files[filePath].deleted = true;
    this.metadataStore.data.files[filePath].deletedAt = Date.now();
    this.metadataStore.save();
  }

  async loadMetadata() {
    await this.metadataStore.load();
    if (Object.keys(this.metadataStore.data.files).length === 0) {
      let files = [];
      let folders = [this.settings.localContentDir];
      while (folders.length > 0) {
        const folder = folders.pop();
        if (folder === undefined) {
          continue;
        }
        if (!this.settings.syncConfigDir && folder === this.vault.configDir) {
          // Skip the config dir if the user doesn't want to sync it
          continue;
        }
        const res = await this.vault.adapter.list(folder);
        files.push(...res.files);
        folders.push(...res.folders);
      }
      files.forEach((filePath: string) => {
        if (filePath === `${this.vault.configDir}/workspace.json`) {
          // Obsidian recommends not syncing the workspace file
          return;
        }

        // We change the remote path only if the file is not in the config dir
        // as that directory is always at the root of the repo.
        // If it's synced obviously.
        let remotePath = filePath;
        if (!filePath.startsWith(this.vault.configDir)) {
          if (this.settings.localContentDir === "") {
            if (this.settings.repoContentDir === "") {
              remotePath = filePath;
            } else {
              remotePath = `${this.settings.repoContentDir}/${filePath}`;
            }
          } else {
            remotePath = filePath.replace(
              this.settings.localContentDir,
              this.settings.repoContentDir,
            );
          }
        }

        this.metadataStore.data.files[filePath] = {
          localPath: filePath,
          remotePath: remotePath,
          sha: null,
          dirty: false,
          justDownloaded: false,
          lastModified: Date.now(),
        };
      });

      // Must be the first time we run, initialize the metadata store
      // with itself and all files in the local content dir.
      this.metadataStore.data.files[
        `${this.vault.configDir}/github-sync-metadata.json`
      ] = {
        localPath: `${this.vault.configDir}/github-sync-metadata.json`,
        remotePath: `${this.vault.configDir}/github-sync-metadata.json`,
        sha: null,
        dirty: false,
        justDownloaded: false,
        lastModified: Date.now(),
      };
      this.metadataStore.save();
    }
  }

  /**
   * Add all the files in the config dir in the metadata store.
   * This is mainly useful when the user changes the sync config settings
   * as we need to add those files to the metadata store or they would never be synced.
   */
  async addConfigDirToMetadata() {
    // Get all the files in the config dir
    let files = [];
    let folders = [this.vault.configDir];
    while (folders.length > 0) {
      const folder = folders.pop();
      if (folder === undefined) {
        continue;
      }
      const res = await this.vault.adapter.list(folder);
      files.push(...res.files);
      folders.push(...res.folders);
    }
    // Add them to the metadata store
    files.forEach((filePath: string) => {
      this.metadataStore.data.files[filePath] = {
        localPath: filePath,
        // Remote path is the same for config dir files
        remotePath: filePath,
        sha: null,
        dirty: false,
        justDownloaded: false,
        lastModified: Date.now(),
      };
    });
    this.metadataStore.save();
  }

  /**
   * Remove all the files in the config dir from the metadata store.
   * The metadata file is not removed as it must always be present.
   * This is mainly useful when the user changes the sync config settings
   * as we need to remove those files to the metadata store or they would
   * keep being synced.
   */
  async removeConfigDirFromMetadata() {
    // Get all the files in the config dir
    let files = [];
    let folders = [this.vault.configDir];
    while (folders.length > 0) {
      const folder = folders.pop();
      if (folder === undefined) {
        continue;
      }
      const res = await this.vault.adapter.list(folder);
      files.push(...res.files);
      folders.push(...res.folders);
    }

    // Remove all them from the metadata store
    files.forEach((filePath: string) => {
      if (filePath === `${this.vault.configDir}/github-sync-metadata.json`) {
        // We don't want to remove the metadata file even if it's in the config dir
        return;
      }
      delete this.metadataStore.data.files[filePath];
    });
    this.metadataStore.save();
  }

  getFileMetadata(filePath: string): FileMetadata {
    return this.metadataStore.data.files[filePath];
  }

  async startEventsListener() {
    this.eventsListener.start();
  }

  /**
   * Starts a new sync interval.
   * Raises an error if the interval is already running.
   */
  startSyncInterval(minutes: number): number {
    if (this.syncIntervalId) {
      throw new Error("Sync interval is already running");
    }
    this.syncIntervalId = window.setInterval(
      () => this.sync(),
      // Sync interval is set in minutes but setInterval expects milliseconds
      minutes * 60 * 1000,
    );
    return this.syncIntervalId;
  }

  /**
   * Stops the currently running sync interval
   */
  stopSyncInterval() {
    if (this.syncIntervalId) {
      window.clearInterval(this.syncIntervalId);
      this.syncIntervalId = null;
    }
  }

  /**
   * Util function that stops and restart the sync interval
   */
  restartSyncInterval(minutes: number) {
    this.stopSyncInterval();
    return this.startSyncInterval(minutes);
  }
}