diff --git a/src/main.ts b/src/main.ts index 55e0d28..647fd96 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,7 +3,7 @@ import { createApp, reactive, type App as VueApp } from "vue"; import { createRandomId, createVaultId, normalizeKey } from "./core/ids"; import SettingsApp from "./settings/SettingsApp.vue"; import "./styles.scss"; -import { syncOnce, type LocalSyncState, type VaultIO } from "./sync/engine"; +import { releaseDeletedContent, syncOnce, type LocalSyncState, type VaultIO } from "./sync/engine"; import { S3ObjectStore } from "./store/s3"; declare const require: ((id: string) => unknown) | undefined; @@ -42,7 +42,7 @@ const DEFAULT_SETTINGS: ObsyncSettings = { region: "", accessKeyId: "", secretAccessKey: "", - rootPrefix: "obsync/v1", + rootPrefix: "obsync/v2", accountKey: "default", vaultKey: "", vaultId: "", @@ -91,6 +91,14 @@ export default class ObsyncPlugin extends Plugin { }, }); + this.addCommand({ + id: "release-deleted-content", + name: "释放已删除内容", + callback: async () => { + await this.releaseDeletedContentNow(); + }, + }); + this.addSettingTab(new ObsyncSettingTab(this.app, this)); this.registerAutoSyncTriggers(); } @@ -104,38 +112,13 @@ export default class ObsyncPlugin extends Plugin { const syncingNotice = new Notice("Obsync 正在同步...", 0); try { - const store = new S3ObjectStore({ - endpoint: this.settings.endpoint, - bucket: this.settings.bucket, - region: this.settings.region || "auto", - accessKeyId: this.settings.accessKeyId, - secretAccessKey: this.settings.secretAccessKey, - rootPrefix: this.settings.rootPrefix, - vaultId: this.settings.vaultId, - deviceId: this.settings.deviceId, - now: () => Date.now(), - request: async (request) => { - const { host: _host, ...headers } = request.headers; - const response = await requestUrl({ - url: request.url, - method: request.method, - headers, - body: request.body, - throw: false, - }); - - return { - status: response.status, - text: response.text, - arrayBuffer: response.arrayBuffer, - }; - }, - }); + const store = this.createObjectStore(); const result = await syncOnce({ vault: new ObsidianVaultIO(this.app), store, state: this.settings.syncState, deviceName: this.settings.deviceName || this.settings.deviceId, + deviceId: this.settings.deviceId, now: () => Date.now(), }); @@ -230,6 +213,72 @@ export default class ObsyncPlugin extends Plugin { return Boolean(this.settings.endpoint && this.settings.bucket && this.settings.accessKeyId && this.settings.secretAccessKey); } + async releaseDeletedContentNow(): Promise { + if (!this.hasConnectionSettings()) { + new Notice("请先填写端点、Bucket、Access Key ID 和 Secret Access Key。"); + return; + } + + const notice = new Notice("Obsync 正在释放已删除内容...", 0); + + try { + const store = this.createObjectStore(); + const result = await releaseDeletedContent({ store, now: () => Date.now() }); + notice.hide(); + + if (result.locked) { + new Notice("另一台设备正在同步,本次释放空间已跳过。"); + return; + } + + this.pruneLocalDeletedState(); + await this.saveSettings(); + new Notice(`Obsync 已释放:清理删除记录 ${result.deletedTombstones},删除 Blob ${result.deletedBlobs}。`); + } catch (error) { + notice.hide(); + const message = error instanceof Error ? error.message : String(error); + new Notice(`释放已删除内容失败:${message}`); + } + } + + private createObjectStore(): S3ObjectStore { + return new S3ObjectStore({ + endpoint: this.settings.endpoint, + bucket: this.settings.bucket, + region: this.settings.region || "auto", + accessKeyId: this.settings.accessKeyId, + secretAccessKey: this.settings.secretAccessKey, + rootPrefix: this.settings.rootPrefix, + vaultId: this.settings.vaultId, + deviceId: this.settings.deviceId, + now: () => Date.now(), + request: async (request) => { + const { host: _host, ...headers } = request.headers; + const response = await requestUrl({ + url: request.url, + method: request.method, + headers, + body: request.body, + throw: false, + }); + + return { + status: response.status, + text: response.text, + arrayBuffer: response.arrayBuffer, + }; + }, + }); + } + + private pruneLocalDeletedState(): void { + for (const [path, fileState] of Object.entries(this.settings.syncState.files)) { + if (fileState.deleted) { + delete this.settings.syncState.files[path]; + } + } + } + async loadSettings(): Promise { const loaded = await this.loadData(); this.settings = Object.assign({}, DEFAULT_SETTINGS, loaded); @@ -238,6 +287,10 @@ export default class ObsyncPlugin extends Plugin { this.settings.vaultKey = normalizeKey(this.app.vault.getName()); } + if (!loaded?.rootPrefix || loaded.rootPrefix === "obsync/v1") { + this.settings.rootPrefix = "obsync/v2"; + } + if (!this.settings.deviceId) { this.settings.deviceId = createRandomId("dev"); } @@ -504,6 +557,9 @@ class ObsyncSettingTab extends PluginSettingTab { new Notice(`导入连接配置失败:${message}`); } }, + onReleaseDeletedContent: async () => { + await this.plugin.releaseDeletedContentNow(); + }, }); this.vueApp.mount(mountEl); } diff --git a/src/settings/SettingsApp.vue b/src/settings/SettingsApp.vue index 4af1c27..5713d2c 100644 --- a/src/settings/SettingsApp.vue +++ b/src/settings/SettingsApp.vue @@ -28,6 +28,7 @@ const emit = defineEmits<{ copyDeviceId: []; copyConnectionConfig: []; pasteConnectionConfig: []; + releaseDeletedContent: []; }>(); const showSecretAccessKey = ref(false); @@ -105,8 +106,8 @@ watchEffect(() => {
@@ -151,6 +152,20 @@ watchEffect(() => { +
+
+
+

释放空间

+

清理已删除文件的远端记录和不再引用的 Blob。

+
+
+ +
+
+
+
diff --git a/src/store/s3.test.ts b/src/store/s3.test.ts index 5bec728..9a55494 100644 --- a/src/store/s3.test.ts +++ b/src/store/s3.test.ts @@ -48,6 +48,69 @@ describe("S3ObjectStore", () => { expect(requests[0]?.method).toBe("DELETE"); expect(requests[0]?.url).toBe("https://s3.example.com/my-bucket/obsync/v1/vaults/vlt_TEST/files/notes/today.md"); }); + + test("lists object keys under the vault prefix", async () => { + const requests: HttpRequest[] = []; + const store = createStore(async (request) => { + requests.push(request); + return { + status: 200, + text: [ + "", + "", + "obsync/v1/vaults/vlt_TEST/blobs/sha256/aa/bb/hash1", + "obsync/v1/vaults/vlt_TEST/blobs/sha256/cc/dd/hash2", + "", + ].join(""), + arrayBuffer: new ArrayBuffer(0), + }; + }); + + await expect(store.listObjectKeys("blobs/sha256/")).resolves.toEqual([ + "blobs/sha256/aa/bb/hash1", + "blobs/sha256/cc/dd/hash2", + ]); + expect(requests[0]?.method).toBe("GET"); + expect(requests[0]?.url).toBe( + "https://s3.example.com/my-bucket?list-type=2&prefix=obsync%2Fv1%2Fvaults%2Fvlt_TEST%2Fblobs%2Fsha256%2F", + ); + }); + + test("lists object keys across paginated responses", async () => { + const requests: HttpRequest[] = []; + const responses = [ + [ + "", + "", + "true", + "page 2", + "obsync/v1/vaults/vlt_TEST/blobs/sha256/aa/bb/hash1", + "", + ].join(""), + [ + "", + "", + "false", + "obsync/v1/vaults/vlt_TEST/blobs/sha256/cc/dd/hash2", + "", + ].join(""), + ]; + const store = createStore(async (request) => { + requests.push(request); + return { + status: 200, + text: responses.shift() ?? "", + arrayBuffer: new ArrayBuffer(0), + }; + }); + + await expect(store.listObjectKeys("blobs/sha256/")).resolves.toEqual([ + "blobs/sha256/aa/bb/hash1", + "blobs/sha256/cc/dd/hash2", + ]); + expect(requests).toHaveLength(2); + expect(requests[1]?.url).toContain("continuation-token=page+2"); + }); }); function createStore(request: (request: HttpRequest) => Promise<{ status: number; text: string; arrayBuffer: ArrayBuffer }>) { diff --git a/src/store/s3.ts b/src/store/s3.ts index 8b671a4..307e462 100644 --- a/src/store/s3.ts +++ b/src/store/s3.ts @@ -96,6 +96,28 @@ export class S3ObjectStore implements ObjectStore { assertOk(response.status, `delete object ${key}`); } + async listObjectKeys(prefix: string): Promise { + const resolvedPrefix = this.resolveKey(prefix); + const keys: string[] = []; + let continuationToken: string | undefined; + + do { + const response = await this.request("GET", "", undefined, { + "list-type": "2", + prefix: resolvedPrefix, + ...(continuationToken ? { "continuation-token": continuationToken } : {}), + }); + assertOk(response.status, `list objects ${prefix}`); + + keys.push(...parseListBucketKeys(response.text)); + continuationToken = readListContinuationToken(response.text); + } while (continuationToken); + + return keys + .filter((key) => key.startsWith(`${this.layout.vaultPrefix}/`)) + .map((key) => key.slice(this.layout.vaultPrefix.length + 1)); + } + private resolveKey(key: string): string { if (key === "manifest.json" || key.startsWith("locks/") || key.startsWith("meta/")) { return `${this.layout.vaultPrefix}/${key}`; @@ -104,8 +126,8 @@ export class S3ObjectStore implements ObjectStore { return `${this.layout.vaultPrefix}/${key}`; } - private async request(method: string, key: string, body?: Uint8Array): Promise { - const url = createObjectUrl(this.options.endpoint, this.options.bucket, key); + private async request(method: string, key: string, body?: Uint8Array, query?: Record): Promise { + const url = createObjectUrl(this.options.endpoint, this.options.bucket, key, query); const bodyBytes = body ?? new Uint8Array(); const headers = await createSignedHeaders({ method, @@ -126,9 +148,11 @@ export class S3ObjectStore implements ObjectStore { } } -function createObjectUrl(endpoint: string, bucket: string, key: string): string { +function createObjectUrl(endpoint: string, bucket: string, key: string, query?: Record): string { const base = endpoint.replace(/\/+$/g, ""); - return `${base}/${encodePathPart(bucket)}/${encodeKey(key)}`; + const path = key ? `${encodePathPart(bucket)}/${encodeKey(key)}` : encodePathPart(bucket); + const search = query ? `?${new URLSearchParams(query).toString()}` : ""; + return `${base}/${path}${search}`; } function encodeKey(key: string): string { @@ -165,7 +189,7 @@ async function createSignedHeaders(input: { const canonicalRequest = [ input.method.toUpperCase(), url.pathname, - url.searchParams.toString(), + createCanonicalQueryString(url), canonicalHeaders, signedHeaders, bodyHash, @@ -210,6 +234,19 @@ function formatAmzDate(timestamp: number): string { return new Date(timestamp).toISOString().replace(/[:-]|\.\d{3}/g, ""); } +function createCanonicalQueryString(url: URL): string { + return Array.from(url.searchParams.entries()) + .sort(([leftKey, leftValue], [rightKey, rightValue]) => { + if (leftKey === rightKey) { + return leftValue.localeCompare(rightValue); + } + + return leftKey.localeCompare(rightKey); + }) + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) + .join("&"); +} + function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { const copy = new Uint8Array(bytes.byteLength); copy.set(bytes); @@ -224,6 +261,36 @@ function safeParseJson(value: string): unknown { } } +function parseListBucketKeys(xml: string): string[] { + const keys: string[] = []; + const keyPattern = /([^<]*)<\/Key>/g; + let match: RegExpExecArray | null; + + while ((match = keyPattern.exec(xml)) !== null) { + keys.push(decodeXml(match[1])); + } + + return keys; +} + +function readListContinuationToken(xml: string): string | undefined { + if (!/true<\/IsTruncated>/i.test(xml)) { + return undefined; + } + + const match = xml.match(/([^<]*)<\/NextContinuationToken>/); + return match ? decodeXml(match[1]) : undefined; +} + +function decodeXml(value: string): string { + return value + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, "\"") + .replace(/'/g, "'") + .replace(/&/g, "&"); +} + function assertOk(status: number, operation: string): void { if (status < 200 || status >= 300) { throw new Error(`Failed to ${operation}: HTTP ${status}`); diff --git a/src/sync/engine.test.ts b/src/sync/engine.test.ts index 19e80ad..f659f04 100644 --- a/src/sync/engine.test.ts +++ b/src/sync/engine.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { createEmptyManifest, syncOnce, type LocalSyncState, type ObjectStore, type VaultIO } from "./engine"; +import { blobObjectKey, createEmptyManifest, releaseDeletedContent, syncOnce, type LocalSyncState, type ObjectStore, type VaultIO } from "./engine"; describe("sync engine", () => { test("uploads a local file when remote has no version", async () => { @@ -7,12 +7,13 @@ describe("sync engine", () => { const store = new FakeObjectStore(); const state: LocalSyncState = { files: {} }; - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 1000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); + const hash = await hashText("hello"); expect(result.uploaded).toBe(1); - expect(await store.getText("files/notes/today.md")).toBe("hello"); - expect(store.manifest.files["notes/today.md"]?.deleted).toBe(false); - expect(state.files["notes/today.md"]?.lastSyncedHash).toBe(store.manifest.files["notes/today.md"]?.hash); + expect(await store.getText(blobObjectKey(hash))).toBe("hello"); + expect(store.manifest.paths["notes/today.md"]?.contentHash).toBe(hash); + expect(state.files["notes/today.md"]?.lastSyncedHash).toBe(hash); }); test("does not read unchanged local files after metadata is cached", async () => { @@ -20,9 +21,9 @@ describe("sync engine", () => { const store = new FakeObjectStore(); const state: LocalSyncState = { files: {} }; - await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 1000 }); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); vault.readCount = 0; - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 2000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); expect(result.uploaded).toBe(0); expect(result.downloaded).toBe(0); @@ -34,11 +35,11 @@ describe("sync engine", () => { const store = new FakeObjectStore(); const state: LocalSyncState = { files: {} }; - await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 1000 }); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); store.acquireLockCount = 0; store.writeManifestCount = 0; - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 2000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); expect(result.locked).toBe(false); expect(store.acquireLockCount).toBe(0); @@ -54,37 +55,147 @@ describe("sync engine", () => { lastSyncedHash: await hashText("base"), remoteHash: await hashText("base"), deleted: false, + version: "ver_base", }, }, }; - await store.putText("files/notes/today.md", "remote"); - store.manifest.files["notes/today.md"] = { - hash: await hashText("remote"), + await store.putText(blobObjectKey(await hashText("remote")), "remote"); + store.manifest.paths["notes/today.md"] = { + contentHash: await hashText("remote"), size: 6, updatedAt: 1000, - deleted: false, + updatedBy: "dev_other", + revision: 1, + version: "ver_remote", }; - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 2000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); expect(result.downloaded).toBe(1); expect(await vault.readText("notes/today.md")).toBe("remote"); expect(state.files["notes/today.md"]?.lastSyncedHash).toBe(await hashText("remote")); }); - test("deletes the remote object when a synced local file is deleted", async () => { + test("syncs a new file from one device to another device", async () => { + const deviceA = new FakeVault({ "notes/shared.md": "from A" }); + const deviceB = new FakeVault({}); + const store = new FakeObjectStore(); + const stateA: LocalSyncState = { files: {} }; + const stateB: LocalSyncState = { files: {} }; + + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 1000 }); + const result = await syncOnce({ vault: deviceB, store, state: stateB, deviceName: "Phone", deviceId: "dev_b", now: () => 2000 }); + + expect(result.downloaded).toBe(1); + expect(await deviceB.readText("notes/shared.md")).toBe("from A"); + expect(stateB.files["notes/shared.md"]?.version).toBe(stateA.files["notes/shared.md"]?.version); + }); + + test("syncs a deletion from one device to another device", async () => { + const deviceA = new FakeVault({ "notes/shared.md": "base" }); + const deviceB = new FakeVault({}); + const store = new FakeObjectStore(); + const stateA: LocalSyncState = { files: {} }; + const stateB: LocalSyncState = { files: {} }; + + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 1000 }); + await syncOnce({ vault: deviceB, store, state: stateB, deviceName: "Phone", deviceId: "dev_b", now: () => 2000 }); + await deviceA.delete("notes/shared.md"); + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 3000 }); + const result = await syncOnce({ vault: deviceB, store, state: stateB, deviceName: "Phone", deviceId: "dev_b", now: () => 4000 }); + + expect(result.deletedLocal).toBe(1); + expect(await deviceB.readText("notes/shared.md")).toBe(""); + expect(stateB.files["notes/shared.md"]?.deleted).toBe(true); + expect(store.manifest.deleted["notes/shared.md"]).toBeDefined(); + }); + + test("syncs delete then same-name recreate as a new version to another device", async () => { + const deviceA = new FakeVault({ "notes/shared.md": "old" }); + const deviceB = new FakeVault({}); + const store = new FakeObjectStore(); + const stateA: LocalSyncState = { files: {} }; + const stateB: LocalSyncState = { files: {} }; + + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 1000 }); + await syncOnce({ vault: deviceB, store, state: stateB, deviceName: "Phone", deviceId: "dev_b", now: () => 2000 }); + const oldVersion = stateB.files["notes/shared.md"]?.version; + await deviceA.delete("notes/shared.md"); + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 3000 }); + await syncOnce({ vault: deviceB, store, state: stateB, deviceName: "Phone", deviceId: "dev_b", now: () => 4000 }); + await deviceA.write("notes/shared.md", new TextEncoder().encode("new")); + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 5000 }); + const result = await syncOnce({ vault: deviceB, store, state: stateB, deviceName: "Phone", deviceId: "dev_b", now: () => 6000 }); + + expect(result.downloaded).toBe(1); + expect(await deviceB.readText("notes/shared.md")).toBe("new"); + expect(stateB.files["notes/shared.md"]?.version).not.toBe(oldVersion); + expect(store.manifest.deleted["notes/shared.md"]?.previousVersion).toBe(oldVersion); + }); + + test("keeps stale device content as a conflict instead of resurrecting a deleted file", async () => { + const deviceA = new FakeVault({ "notes/shared.md": "base" }); + const staleDevice = new FakeVault({ "notes/shared.md": "stale local" }); + const store = new FakeObjectStore(); + const stateA: LocalSyncState = { files: {} }; + const staleState: LocalSyncState = { files: {} }; + + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 1000 }); + await deviceA.delete("notes/shared.md"); + await syncOnce({ vault: deviceA, store, state: stateA, deviceName: "Mac", deviceId: "dev_a", now: () => 2000 }); + const result = await syncOnce({ vault: staleDevice, store, state: staleState, deviceName: "Phone", deviceId: "dev_b", now: () => 3000 }); + + expect(result.conflicts).toBe(1); + expect(result.uploaded).toBe(0); + expect(store.manifest.paths["notes/shared.md"]).toBeUndefined(); + expect(await staleDevice.readText("notes/shared.md")).toBe(""); + expect(await staleDevice.readText("notes/shared.conflict.Phone.19700101-080003.md")).toBe("stale local"); + }); + + test("stores uploaded content under a hash blob instead of the original path", async () => { const vault = new FakeVault({ "notes/today.md": "hello" }); const store = new FakeObjectStore(); const state: LocalSyncState = { files: {} }; - await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 1000 }); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); + + const hash = await hashText("hello"); + expect(await store.getText(`blobs/sha256/${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash}`)).toBe("hello"); + expect(store.objects["files/notes/today.md"]).toBeUndefined(); + expect(store.manifest.paths["notes/today.md"]?.contentHash).toBe(hash); + }); + + test("rewrites a hash blob when manifest has a stale blob record but the object is missing", async () => { + const vault = new FakeVault({ "notes/today.md": "hello" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { files: {} }; + const hash = await hashText("hello"); + store.manifest.blobs[hash] = { + key: blobObjectKey(hash), + size: 5, + createdAt: 1000, + }; + + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); + + expect(await store.getText(blobObjectKey(hash))).toBe("hello"); + }); + + test("marks a path deleted without immediately deleting the old blob", async () => { + const vault = new FakeVault({ "notes/today.md": "hello" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { files: {} }; + + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); + const hash = await hashText("hello"); await vault.delete("notes/today.md"); - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 2000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); expect(result.deletedRemote).toBe(1); expect(result.conflicts).toBe(0); - expect(store.objects["files/notes/today.md"]).toBeUndefined(); - expect(store.manifest.files["notes/today.md"]?.deleted).toBe(true); + expect(await store.getText(`blobs/sha256/${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash}`)).toBe("hello"); + expect(store.manifest.paths["notes/today.md"]).toBeUndefined(); + expect(store.manifest.deleted["notes/today.md"]?.previousContentHash).toBe(hash); }); test("local deletion wins instead of creating a conflict", async () => { @@ -97,23 +208,108 @@ describe("sync engine", () => { lastSyncedHash: baseHash, remoteHash: baseHash, deleted: false, + version: "ver_base", }, }, }; - await store.putText("files/notes/today.md", "remote"); - store.manifest.files["notes/today.md"] = { - hash: await hashText("remote"), + await store.putText(blobObjectKey(await hashText("remote")), "remote"); + store.manifest.paths["notes/today.md"] = { + contentHash: await hashText("remote"), size: 6, updatedAt: 1000, - deleted: false, + updatedBy: "dev_other", + revision: 1, + version: "ver_remote", }; - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 2000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); expect(result.deletedRemote).toBe(1); expect(result.conflicts).toBe(0); - expect(store.objects["files/notes/today.md"]).toBeUndefined(); - expect(store.manifest.files["notes/today.md"]?.deleted).toBe(true); + expect(store.manifest.paths["notes/today.md"]).toBeUndefined(); + expect(store.manifest.deleted["notes/today.md"]?.previousContentHash).toBe(baseHash); + }); + + test("deletes local file when another device deleted the path", async () => { + const hash = await hashText("base"); + const vault = new FakeVault({ "notes/today.md": "base" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { + files: { + "notes/today.md": { + lastSyncedHash: hash, + remoteHash: hash, + deleted: false, + version: "ver_base", + }, + }, + }; + store.manifest.deleted["notes/today.md"] = { + deletedAt: 2000, + deletedRevision: 2, + previousContentHash: hash, + previousVersion: "ver_base", + deletedBy: "dev_other", + }; + + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 3000 }); + + expect(result.deletedLocal).toBe(1); + await expect(vault.readText("notes/today.md")).resolves.toBe(""); + expect(state.files["notes/today.md"]?.deleted).toBe(true); + }); + + test("keeps a conflict copy when remote deleted a path that changed locally", async () => { + const hash = await hashText("base"); + const vault = new FakeVault({ "notes/today.md": "local changed" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { + files: { + "notes/today.md": { + lastSyncedHash: hash, + remoteHash: hash, + deleted: false, + version: "ver_base", + }, + }, + }; + store.manifest.deleted["notes/today.md"] = { + deletedAt: 2000, + deletedRevision: 2, + previousContentHash: hash, + previousVersion: "ver_base", + deletedBy: "dev_other", + }; + + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 3000 }); + + expect(result.conflicts).toBe(1); + expect(result.deletedLocal).toBe(1); + expect(await vault.readText("notes/today.md")).toBe(""); + expect(await vault.readText("notes/today.conflict.Mac.19700101-080003.md")).toBe("local changed"); + expect(state.files["notes/today.md"]?.deleted).toBe(true); + }); + + test("does not resurrect a tombstoned path when local state is missing", async () => { + const hash = await hashText("deleted"); + const vault = new FakeVault({ "notes/today.md": "stale local" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { files: {} }; + store.manifest.deleted["notes/today.md"] = { + deletedAt: 2000, + deletedRevision: 2, + previousContentHash: hash, + previousVersion: "ver_deleted", + deletedBy: "dev_other", + }; + + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 3000 }); + + expect(result.conflicts).toBe(1); + expect(result.uploaded).toBe(0); + expect(store.manifest.paths["notes/today.md"]).toBeUndefined(); + expect(await vault.readText("notes/today.md")).toBe(""); + expect(await vault.readText("notes/today.conflict.Mac.19700101-080003.md")).toBe("stale local"); }); test("keeps recent tombstones in the manifest", async () => { @@ -125,20 +321,22 @@ describe("sync engine", () => { lastSyncedHash: "old", remoteHash: "old", deleted: true, + version: "ver_old", }, }, }; - store.manifest.files["notes/today.md"] = { - hash: "old", - size: 0, - updatedAt: 1000, - deleted: true, + store.manifest.deleted["notes/today.md"] = { + deletedAt: 1000, + deletedRevision: 1, + previousContentHash: "old", + previousVersion: "ver_old", + deletedBy: "dev_other", }; store.writeManifestCount = 0; - await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 1000 + 29 * 24 * 60 * 60 * 1000 }); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 + 29 * 24 * 60 * 60 * 1000 }); - expect(store.manifest.files["notes/today.md"]?.deleted).toBe(true); + expect(store.manifest.deleted["notes/today.md"]).toBeDefined(); expect(state.files["notes/today.md"]?.deleted).toBe(true); expect(store.writeManifestCount).toBe(0); }); @@ -152,23 +350,235 @@ describe("sync engine", () => { lastSyncedHash: "old", remoteHash: "old", deleted: true, + version: "ver_old", }, }, }; - store.manifest.files["notes/today.md"] = { - hash: "old", - size: 0, - updatedAt: 1000, - deleted: true, + store.manifest.deleted["notes/today.md"] = { + deletedAt: 1000, + deletedRevision: 1, + previousContentHash: "old", + previousVersion: "ver_old", + deletedBy: "dev_other", }; - await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 1000 + 30 * 24 * 60 * 60 * 1000 }); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 + 30 * 24 * 60 * 60 * 1000 }); - expect(store.manifest.files["notes/today.md"]).toBeUndefined(); + expect(store.manifest.deleted["notes/today.md"]).toBeUndefined(); expect(state.files["notes/today.md"]).toBeUndefined(); expect(store.writeManifestCount).toBe(1); }); + test("pruning an old tombstone does not remove current state for a same-name recreated file", async () => { + const activeHash = await hashText("new"); + const vault = new FakeVault({ "notes/today.md": "new" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { + files: { + "notes/today.md": { + lastSyncedHash: activeHash, + remoteHash: activeHash, + deleted: false, + version: "ver_new", + localMtime: 1, + localSize: 3, + }, + }, + }; + await store.putText(blobObjectKey(activeHash), "new"); + store.manifest.paths["notes/today.md"] = { + contentHash: activeHash, + size: 3, + updatedAt: 2000, + updatedBy: "dev_mac", + revision: 2, + version: "ver_new", + }; + store.manifest.deleted["notes/today.md"] = { + deletedAt: 1000, + deletedRevision: 1, + previousContentHash: await hashText("old"), + previousVersion: "ver_old", + deletedBy: "dev_mac", + }; + + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 + 30 * 24 * 60 * 60 * 1000 }); + + expect(result.uploaded).toBe(0); + expect(store.manifest.deleted["notes/today.md"]).toBeUndefined(); + expect(store.manifest.paths["notes/today.md"]?.version).toBe("ver_new"); + expect(state.files["notes/today.md"]?.version).toBe("ver_new"); + expect(state.files["notes/today.md"]?.deleted).toBe(false); + }); + + test("treats delete then same-name create as a new version", async () => { + const vault = new FakeVault({ "notes/today.md": "old" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { files: {} }; + + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); + const oldVersion = store.manifest.paths["notes/today.md"]?.version; + await vault.delete("notes/today.md"); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); + await vault.write("notes/today.md", new TextEncoder().encode("new")); + + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 3000 }); + + expect(result.uploaded).toBe(1); + expect(store.manifest.paths["notes/today.md"]?.contentHash).toBe(await hashText("new")); + expect(store.manifest.paths["notes/today.md"]?.version).not.toBe(oldVersion); + expect(store.manifest.deleted["notes/today.md"]?.previousVersion).toBe(oldVersion); + expect(state.files["notes/today.md"]?.deleted).toBe(false); + }); + + test("same-name create gets a new version even when content matches the deleted file", async () => { + const vault = new FakeVault({ "notes/today.md": "same" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { files: {} }; + + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 1000 }); + const oldVersion = store.manifest.paths["notes/today.md"]?.version; + await vault.delete("notes/today.md"); + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); + await vault.write("notes/today.md", new TextEncoder().encode("same")); + + await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 3000 }); + + expect(store.manifest.paths["notes/today.md"]?.contentHash).toBe(await hashText("same")); + expect(store.manifest.paths["notes/today.md"]?.version).not.toBe(oldVersion); + }); + + test("writes a conflict copy when a deleted path was recreated remotely and local also changed", async () => { + const baseHash = await hashText("base"); + const remoteHash = await hashText("remote-new"); + const vault = new FakeVault({ "notes/today.md": "local-new" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { + files: { + "notes/today.md": { + lastSyncedHash: baseHash, + remoteHash: baseHash, + deleted: false, + version: "ver_base", + }, + }, + }; + await store.putText(blobObjectKey(remoteHash), "remote-new"); + store.manifest.paths["notes/today.md"] = { + contentHash: remoteHash, + size: 10, + updatedAt: 3000, + updatedBy: "dev_other", + revision: 3, + version: "ver_remote_recreated", + }; + store.manifest.deleted["notes/today.md"] = { + deletedAt: 2000, + deletedRevision: 2, + previousContentHash: baseHash, + previousVersion: "ver_base", + deletedBy: "dev_other", + }; + + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 4000 }); + + expect(result.conflicts).toBe(1); + expect(await vault.readText("notes/today.md")).toBe("local-new"); + expect(await vault.readText("notes/today.conflict.Mac.19700101-080004.md")).toBe("remote-new"); + }); + + test("does not create duplicate conflict copies after the same remote conflict was already recorded", async () => { + const baseHash = await hashText("base"); + const remoteHash = await hashText("remote"); + const vault = new FakeVault({ "notes/today.md": "local" }); + const store = new FakeObjectStore(); + const state: LocalSyncState = { + files: { + "notes/today.md": { + lastSyncedHash: baseHash, + remoteHash: baseHash, + deleted: false, + version: "ver_base", + }, + }, + }; + await store.putText(blobObjectKey(remoteHash), "remote"); + store.manifest.paths["notes/today.md"] = { + contentHash: remoteHash, + size: 6, + updatedAt: 2000, + updatedBy: "dev_other", + revision: 2, + version: "ver_remote", + }; + + const firstResult = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 3000 }); + const secondResult = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 4000 }); + + expect(firstResult.conflicts).toBe(1); + expect(secondResult.conflicts).toBe(0); + expect(secondResult.uploaded).toBe(2); + expect(await vault.readText("notes/today.conflict.Mac.19700101-080003.md")).toBe("remote"); + expect(await vault.readText("notes/today.conflict.Mac.19700101-080004.md")).toBe(""); + expect(store.manifest.paths["notes/today.md"]?.contentHash).toBe(await hashText("local")); + }); + + test("releaseDeletedContent removes deleted tombstones and unreferenced blobs", async () => { + const keptHash = await hashText("kept"); + const deletedHash = await hashText("deleted"); + const orphanHash = await hashText("orphan"); + const store = new FakeObjectStore(); + store.manifest.paths["notes/kept.md"] = { + contentHash: keptHash, + size: 4, + updatedAt: 1000, + updatedBy: "dev_mac", + revision: 1, + version: "ver_kept", + }; + store.manifest.deleted["notes/deleted.md"] = { + deletedAt: 1000, + deletedRevision: 2, + previousContentHash: deletedHash, + previousVersion: "ver_deleted", + deletedBy: "dev_mac", + }; + await store.putText(blobObjectKey(keptHash), "kept"); + await store.putText(blobObjectKey(deletedHash), "deleted"); + await store.putText(blobObjectKey(orphanHash), "orphan"); + + const result = await releaseDeletedContent({ store, now: () => 5000 }); + + expect(result.deletedTombstones).toBe(1); + expect(result.deletedBlobs).toBe(2); + expect(store.manifest.deleted["notes/deleted.md"]).toBeUndefined(); + expect(store.objects[blobObjectKey(keptHash)]).toBeDefined(); + expect(store.objects[blobObjectKey(deletedHash)]).toBeUndefined(); + expect(store.objects[blobObjectKey(orphanHash)]).toBeUndefined(); + }); + + test("releaseDeletedContent skips cleanup when the remote lock is held", async () => { + const deletedHash = await hashText("deleted"); + const store = new FakeObjectStore(); + store.lockAvailable = false; + store.manifest.deleted["notes/deleted.md"] = { + deletedAt: 1000, + deletedRevision: 2, + previousContentHash: deletedHash, + previousVersion: "ver_deleted", + deletedBy: "dev_mac", + }; + await store.putText(blobObjectKey(deletedHash), "deleted"); + + const result = await releaseDeletedContent({ store, now: () => 5000 }); + + expect(result.locked).toBe(true); + expect(result.deletedTombstones).toBe(0); + expect(result.deletedBlobs).toBe(0); + expect(store.manifest.deleted["notes/deleted.md"]).toBeDefined(); + expect(store.objects[blobObjectKey(deletedHash)]).toBeDefined(); + }); + test("writes a conflict copy when local and remote both changed", async () => { const baseHash = await hashText("base"); const vault = new FakeVault({ "notes/today.md": "local" }); @@ -179,18 +589,21 @@ describe("sync engine", () => { lastSyncedHash: baseHash, remoteHash: baseHash, deleted: false, + version: "ver_base", }, }, }; - await store.putText("files/notes/today.md", "remote"); - store.manifest.files["notes/today.md"] = { - hash: await hashText("remote"), + await store.putText(blobObjectKey(await hashText("remote")), "remote"); + store.manifest.paths["notes/today.md"] = { + contentHash: await hashText("remote"), size: 6, updatedAt: 1000, - deleted: false, + updatedBy: "dev_other", + revision: 1, + version: "ver_remote", }; - const result = await syncOnce({ vault, store, state, deviceName: "Mac", now: () => 2000 }); + const result = await syncOnce({ vault, store, state, deviceName: "Mac", deviceId: "dev_mac", now: () => 2000 }); expect(result.conflicts).toBe(1); expect(await vault.readText("notes/today.md")).toBe("local"); @@ -245,11 +658,12 @@ class FakeObjectStore implements ObjectStore { manifest = createEmptyManifest(); objects: Record = {}; acquireLockCount = 0; + lockAvailable = true; writeManifestCount = 0; async acquireLock(): Promise { this.acquireLockCount += 1; - return true; + return this.lockAvailable; } async releaseLock(): Promise { @@ -277,6 +691,10 @@ class FakeObjectStore implements ObjectStore { delete this.objects[key]; } + async listObjectKeys(prefix: string): Promise { + return Object.keys(this.objects).filter((key) => key.startsWith(prefix)); + } + async putText(key: string, value: string): Promise { await this.writeObject(key, new TextEncoder().encode(value)); } diff --git a/src/sync/engine.ts b/src/sync/engine.ts index a031437..8913ead 100644 --- a/src/sync/engine.ts +++ b/src/sync/engine.ts @@ -1,23 +1,41 @@ -import { planFileAction } from "./planner"; -import type { SyncAction } from "./planner"; - export interface ManifestFileEntry { - hash: string; + contentHash: string; size: number; updatedAt: number; - deleted: boolean; + updatedBy: string; + revision: number; + version: string; +} + +export interface DeletedPathEntry { + deletedAt: number; + deletedRevision: number; + previousContentHash: string | null; + previousVersion: string | null; + deletedBy: string; +} + +export interface BlobEntry { + key: string; + size: number; + createdAt: number; } export interface RemoteManifest { - schemaVersion: 1; + schemaVersion: 2; + revision: number; updatedAt: number; - files: Record; + paths: Record; + deleted: Record; + blobs: Record; + devices: Record; } export interface LocalFileState { lastSyncedHash: string | null; remoteHash: string | null; deleted: boolean; + version?: string | null; localMtime?: number | null; localSize?: number | null; } @@ -42,6 +60,7 @@ export interface ObjectStore { readObject(key: string): Promise; writeObject(key: string, bytes: Uint8Array): Promise; deleteObject(key: string): Promise; + listObjectKeys?(prefix: string): Promise; } export interface SyncResult { @@ -58,6 +77,7 @@ interface SyncOnceInput { store: ObjectStore; state: LocalSyncState; deviceName: string; + deviceId: string; now(): number; } @@ -66,17 +86,25 @@ interface PlannedFile { action: SyncAction; local?: { bytes: Uint8Array; hash: string; mtime: number; size: number }; remote?: ManifestFileEntry; + deleted?: DeletedPathEntry; lastSyncedHash: string | null; + lastVersion: string | null; } +type SyncAction = "noop" | "upload" | "download" | "conflict" | "mark-remote-deleted" | "delete-local"; + const SYNC_CONCURRENCY = 4; const TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; export function createEmptyManifest(): RemoteManifest { return { - schemaVersion: 1, + schemaVersion: 2, + revision: 0, updatedAt: 0, - files: {}, + paths: {}, + deleted: {}, + blobs: {}, + devices: {}, }; } @@ -89,7 +117,7 @@ export async function syncOnce(input: SyncOnceInput): Promise { deletedRemote: 0, locked: false, }; - const manifest = await input.store.readManifest(); + const manifest = normalizeManifest(await input.store.readManifest()); const localFiles = await scanLocalFiles(input.vault, input.state); const plan = createPlan(localFiles, manifest, input.state); const actions = plan.filter((item) => item.action !== "noop"); @@ -98,6 +126,7 @@ export async function syncOnce(input: SyncOnceInput): Promise { actions.some((item) => item.action === "upload" || item.action === "mark-remote-deleted") || expiredTombstones.length > 0; if (actions.length === 0 && expiredTombstones.length === 0) { + updateDeviceCheckpoint(manifest, input); input.state.lastSyncAt = input.now(); return result; } @@ -116,49 +145,70 @@ export async function syncOnce(input: SyncOnceInput): Promise { try { await mapLimit(actions, SYNC_CONCURRENCY, async (item) => { if (item.action === "upload" && item.local) { - await input.store.writeObject(fileObjectKey(item.path), item.local.bytes); - manifest.files[item.path] = { - hash: item.local.hash, + await writeBlob(input.store, manifest, item.local.hash, item.local.bytes, item.local.size, input.now()); + manifest.paths[item.path] = { + contentHash: item.local.hash, size: item.local.size, updatedAt: input.now(), - deleted: false, + updatedBy: input.deviceId, + revision: nextRevision(manifest), + version: createVersion(input.deviceId, input.now(), item.local.hash), }; - input.state.files[item.path] = syncedState(item.local.hash, item.local); + input.state.files[item.path] = syncedState(item.local.hash, item.local, manifest.paths[item.path].version); result.uploaded += 1; } if (item.action === "download" && item.remote) { - const bytes = await input.store.readObject(fileObjectKey(item.path)); + const bytes = await input.store.readObject(blobObjectKey(item.remote.contentHash)); await input.vault.write(item.path, bytes); - input.state.files[item.path] = syncedState(item.remote.hash, { size: bytes.byteLength, mtime: null }); + input.state.files[item.path] = syncedState(item.remote.contentHash, { size: bytes.byteLength, mtime: null }, item.remote.version); result.downloaded += 1; } if (item.action === "conflict" && item.remote) { - const bytes = await input.store.readObject(fileObjectKey(item.path)); + const bytes = await input.store.readObject(blobObjectKey(item.remote.contentHash)); await input.vault.write(createConflictPath(item.path, input.deviceName, input.now()), bytes); input.state.files[item.path] = { lastSyncedHash: item.lastSyncedHash, - remoteHash: item.remote.hash, + remoteHash: item.remote.contentHash, deleted: false, - localMtime: item.local?.mtime ?? null, - localSize: item.local?.size ?? null, + version: item.remote.version, + localMtime: null, + localSize: null, }; result.conflicts += 1; } - if (item.action === "mark-remote-deleted") { - await input.store.deleteObject(fileObjectKey(item.path)); - manifest.files[item.path] = { - hash: item.lastSyncedHash ?? "", - size: 0, - updatedAt: input.now(), + if (item.action === "conflict" && item.deleted && item.local && !item.remote) { + await input.vault.write(createConflictPath(item.path, input.deviceName, input.now()), item.local.bytes); + await input.vault.delete(item.path); + input.state.files[item.path] = { + lastSyncedHash: item.deleted.previousContentHash, + remoteHash: item.deleted.previousContentHash, deleted: true, + version: item.deleted.previousVersion, + localMtime: null, + localSize: null, + }; + result.conflicts += 1; + result.deletedLocal += 1; + } + + if (item.action === "mark-remote-deleted") { + const remote = manifest.paths[item.path]; + delete manifest.paths[item.path]; + manifest.deleted[item.path] = { + deletedAt: input.now(), + deletedRevision: nextRevision(manifest), + previousContentHash: item.lastSyncedHash ?? remote?.contentHash ?? null, + previousVersion: item.lastVersion ?? remote?.version ?? null, + deletedBy: input.deviceId, }; input.state.files[item.path] = { - lastSyncedHash: item.lastSyncedHash, - remoteHash: item.lastSyncedHash, + lastSyncedHash: item.lastSyncedHash ?? remote?.contentHash ?? null, + remoteHash: item.lastSyncedHash ?? remote?.contentHash ?? null, deleted: true, + version: item.lastVersion ?? remote?.version ?? null, localMtime: null, localSize: null, }; @@ -168,9 +218,23 @@ export async function syncOnce(input: SyncOnceInput): Promise { if (item.action === "delete-local" && item.remote) { await input.vault.delete(item.path); input.state.files[item.path] = { - lastSyncedHash: item.remote.hash, - remoteHash: item.remote.hash, + lastSyncedHash: item.remote.contentHash, + remoteHash: item.remote.contentHash, deleted: true, + version: item.remote.version, + localMtime: null, + localSize: null, + }; + result.deletedLocal += 1; + } + + if (item.action === "delete-local" && item.deleted) { + await input.vault.delete(item.path); + input.state.files[item.path] = { + lastSyncedHash: item.deleted.previousContentHash, + remoteHash: item.deleted.previousContentHash, + deleted: true, + version: item.deleted.previousVersion, localMtime: null, localSize: null, }; @@ -179,6 +243,7 @@ export async function syncOnce(input: SyncOnceInput): Promise { }); pruneTombstones(manifest, input.state, expiredTombstones); + updateDeviceCheckpoint(manifest, input); input.state.lastSyncAt = input.now(); if (writesRemote) { @@ -194,16 +259,62 @@ export async function syncOnce(input: SyncOnceInput): Promise { } } +export interface ReleaseDeletedContentResult { + deletedTombstones: number; + deletedBlobs: number; + locked: boolean; +} + +export async function releaseDeletedContent(input: { store: ObjectStore; now(): number }): Promise { + const locked = await input.store.acquireLock(); + + if (!locked) { + return { deletedTombstones: 0, deletedBlobs: 0, locked: true }; + } + + try { + const manifest = normalizeManifest(await input.store.readManifest()); + const deletedTombstones = Object.keys(manifest.deleted).length; + + manifest.deleted = {}; + const referencedHashes = new Set(Object.values(manifest.paths).map((entry) => entry.contentHash)); + const knownBlobKeys = new Set(Object.values(manifest.blobs).map((entry) => entry.key)); + const listedBlobKeys = input.store.listObjectKeys ? await input.store.listObjectKeys("blobs/sha256/") : []; + let deletedBlobs = 0; + + for (const key of new Set([...knownBlobKeys, ...listedBlobKeys])) { + const hash = hashFromBlobKey(key); + + if (hash && !referencedHashes.has(hash)) { + await input.store.deleteObject(key); + delete manifest.blobs[hash]; + deletedBlobs += 1; + } + } + + manifest.revision += 1; + manifest.updatedAt = input.now(); + await input.store.writeManifest(manifest); + + return { deletedTombstones, deletedBlobs, locked: false }; + } finally { + await input.store.releaseLock(); + } +} + function findExpiredTombstones(manifest: RemoteManifest, now: number): string[] { - return Object.entries(manifest.files) - .filter(([, entry]) => entry.deleted && now - entry.updatedAt >= TOMBSTONE_RETENTION_MS) + return Object.entries(manifest.deleted) + .filter(([, entry]) => now - entry.deletedAt >= TOMBSTONE_RETENTION_MS) .map(([path]) => path); } function pruneTombstones(manifest: RemoteManifest, state: LocalSyncState, paths: string[]): void { for (const path of paths) { - delete manifest.files[path]; - delete state.files[path]; + delete manifest.deleted[path]; + + if (state.files[path]?.deleted) { + delete state.files[path]; + } } } @@ -212,47 +323,141 @@ function createPlan( manifest: RemoteManifest, state: LocalSyncState, ): PlannedFile[] { - const paths = new Set([...Object.keys(localFiles), ...Object.keys(manifest.files), ...Object.keys(state.files)]); + const paths = new Set([...Object.keys(localFiles), ...Object.keys(manifest.paths), ...Object.keys(manifest.deleted), ...Object.keys(state.files)]); const plan: PlannedFile[] = []; for (const path of paths) { const local = localFiles[path]; - const remote = manifest.files[path]; + const remote = manifest.paths[path]; + const deleted = manifest.deleted[path]; const previous = state.files[path]; const lastSyncedHash = previous?.lastSyncedHash ?? null; + const lastVersion = previous?.version ?? null; + const action = planPathAction({ local, remote, deleted, previous }); plan.push({ path, local, remote, + deleted, lastSyncedHash, - action: planFileAction({ - localHash: local?.hash ?? null, - lastSyncedHash, - remoteHash: remote?.hash ?? null, - localDeleted: Boolean(previous) && !local, - remoteDeleted: remote?.deleted ?? false, - }), + lastVersion, + action, }); } return plan; } -function fileObjectKey(path: string): string { - return `files/${path}`; +function planPathAction(input: { + local?: { bytes: Uint8Array; hash: string; mtime: number; size: number }; + remote?: ManifestFileEntry; + deleted?: DeletedPathEntry; + previous?: LocalFileState; +}): SyncAction { + if (!input.local) { + if (input.previous && !input.previous.deleted) { + return "mark-remote-deleted"; + } + + if (input.remote) { + return "download"; + } + + return "noop"; + } + + if (input.remote) { + const localChanged = !input.previous || input.previous.deleted || input.local.hash !== input.previous.lastSyncedHash; + const remoteChanged = input.remote.version !== input.previous?.version || input.remote.contentHash !== input.previous?.remoteHash; + + if (!localChanged && !remoteChanged) { + return "noop"; + } + + if (localChanged && !remoteChanged) { + return "upload"; + } + + if (!localChanged && remoteChanged) { + return "download"; + } + + return "conflict"; + } + + if (input.deleted) { + if (input.previous?.deleted) { + return "upload"; + } + + return input.previous && input.local.hash === input.previous.lastSyncedHash ? "delete-local" : "conflict"; + } + + return "upload"; } -function syncedState(hash: string, local: { mtime: number | null; size: number | null }): LocalFileState { +export function blobObjectKey(hash: string): string { + return `blobs/sha256/${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash}`; +} + +function syncedState(hash: string, local: { mtime: number | null; size: number | null }, version: string): LocalFileState { return { lastSyncedHash: hash, remoteHash: hash, deleted: false, + version, localMtime: local.mtime, localSize: local.size, }; } +async function writeBlob(store: ObjectStore, manifest: RemoteManifest, hash: string, bytes: Uint8Array, size: number, now: number): Promise { + const key = blobObjectKey(hash); + + await store.writeObject(key, bytes); + manifest.blobs[hash] = { + key, + size, + createdAt: manifest.blobs[hash]?.createdAt ?? now, + }; +} + +function nextRevision(manifest: RemoteManifest): number { + manifest.revision += 1; + return manifest.revision; +} + +function createVersion(deviceId: string, now: number, hash: string): string { + return `ver_${now}_${deviceId}_${hash.slice(0, 12)}`; +} + +function updateDeviceCheckpoint(manifest: RemoteManifest, input: SyncOnceInput): void { + manifest.devices[input.deviceId] = { + name: input.deviceName, + lastSeenAt: input.now(), + lastSeenRevision: manifest.revision, + }; +} + +function normalizeManifest(manifest: RemoteManifest): RemoteManifest { + if (manifest.schemaVersion === 2) { + manifest.paths ??= {}; + manifest.deleted ??= {}; + manifest.blobs ??= {}; + manifest.devices ??= {}; + manifest.revision ??= 0; + return manifest; + } + + return createEmptyManifest(); +} + +function hashFromBlobKey(key: string): string | null { + const match = key.match(/blobs\/sha256\/[0-9a-f]{2}\/[0-9a-f]{2}\/([0-9a-f]{64})$/); + return match?.[1] ?? null; +} + async function scanLocalFiles( vault: VaultIO, state: LocalSyncState, diff --git a/src/sync/planner.test.ts b/src/sync/planner.test.ts deleted file mode 100644 index 22c79eb..0000000 --- a/src/sync/planner.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { planFileAction } from "./planner"; - -describe("sync planner", () => { - test("uploads when local changed and remote stayed at the last synced hash", () => { - expect( - planFileAction({ - localHash: "local-new", - lastSyncedHash: "base", - remoteHash: "base", - localDeleted: false, - remoteDeleted: false, - }), - ).toBe("upload"); - }); - - test("downloads when remote changed and local stayed at the last synced hash", () => { - expect( - planFileAction({ - localHash: "base", - lastSyncedHash: "base", - remoteHash: "remote-new", - localDeleted: false, - remoteDeleted: false, - }), - ).toBe("download"); - }); - - test("creates a conflict when local and remote changed from the last synced hash", () => { - expect( - planFileAction({ - localHash: "local-new", - lastSyncedHash: "base", - remoteHash: "remote-new", - localDeleted: false, - remoteDeleted: false, - }), - ).toBe("conflict"); - }); - - test("marks remote deleted when local deletion is the only change", () => { - expect( - planFileAction({ - localHash: null, - lastSyncedHash: "base", - remoteHash: "base", - localDeleted: true, - remoteDeleted: false, - }), - ).toBe("mark-remote-deleted"); - }); -}); - diff --git a/src/sync/planner.ts b/src/sync/planner.ts deleted file mode 100644 index 9e223d7..0000000 --- a/src/sync/planner.ts +++ /dev/null @@ -1,32 +0,0 @@ -export type SyncAction = "noop" | "upload" | "download" | "conflict" | "mark-remote-deleted" | "delete-local"; - -interface PlanFileActionInput { - localHash: string | null; - lastSyncedHash: string | null; - remoteHash: string | null; - localDeleted: boolean; - remoteDeleted: boolean; -} - -export function planFileAction(input: PlanFileActionInput): SyncAction { - if (input.localDeleted) { - return input.remoteDeleted ? "noop" : "mark-remote-deleted"; - } - - const localChanged = input.localDeleted || input.localHash !== input.lastSyncedHash; - const remoteChanged = input.remoteDeleted || input.remoteHash !== input.lastSyncedHash; - - if (!localChanged && !remoteChanged) { - return "noop"; - } - - if (localChanged && !remoteChanged) { - return input.localDeleted ? "mark-remote-deleted" : "upload"; - } - - if (!localChanged && remoteChanged) { - return input.remoteDeleted ? "delete-local" : "download"; - } - - return "conflict"; -}