mirror of
https://github.com/ichaly/cohere.git
synced 2026-07-22 07:47:20 +00:00
feat: support configurable S3 addressing style
This commit is contained in:
shi.changliang
parent
20cfda7203
commit
ef414c959d
9 changed files with 192 additions and 10 deletions
13
README.md
13
README.md
|
|
@ -28,12 +28,24 @@ Cohere 用于通过 OSS / S3 兼容对象存储同步当前 vault 文件。
|
|||
|
||||
同步端点填写服务商提供的 **S3 API Endpoint**。
|
||||
|
||||
地址风格默认选择 `自动`。如果服务商明确要求虚拟主机风格地址，请选择 `Virtual Hosted Style`。
|
||||
|
||||
### Cloudflare R2
|
||||
|
||||
```text
|
||||
Endpoint: https://<account-id>.r2.cloudflarestorage.com
|
||||
Region: auto
|
||||
Bucket: <桶名称>
|
||||
地址风格: 自动
|
||||
```
|
||||
|
||||
### 阿里云 OSS
|
||||
|
||||
```text
|
||||
Endpoint: https://oss-<region>.aliyuncs.com
|
||||
Region: oss-<region>
|
||||
Bucket: <桶名称>
|
||||
地址风格: Virtual Hosted Style
|
||||
```
|
||||
|
||||
### 七牛云
|
||||
|
|
@ -42,6 +54,7 @@ Bucket: <桶名称>
|
|||
Endpoint: https://s3.<region>.qiniucs.com
|
||||
Region: <region>
|
||||
Bucket: <桶名称>
|
||||
地址风格: 自动
|
||||
```
|
||||
|
||||
## 本地开发
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ describe("connection config import and export", () => {
|
|||
schemaVersion: 1,
|
||||
endpoint: "https://example.com",
|
||||
bucket: "vault",
|
||||
addressingStyle: "auto",
|
||||
region: "auto",
|
||||
rootPrefix: "obsync/v2",
|
||||
accountKey: "team",
|
||||
|
|
@ -175,6 +176,7 @@ describe("connection config import and export", () => {
|
|||
schemaVersion: 1,
|
||||
endpoint: "https://r2.example.com",
|
||||
bucket: "shared-vault",
|
||||
addressingStyle: "virtual-hosted",
|
||||
region: "auto",
|
||||
rootPrefix: "obsync/v2",
|
||||
accountKey: "team",
|
||||
|
|
@ -185,6 +187,7 @@ describe("connection config import and export", () => {
|
|||
expect(plugin.settings).toMatchObject({
|
||||
endpoint: "https://r2.example.com",
|
||||
bucket: "shared-vault",
|
||||
addressingStyle: "virtual-hosted",
|
||||
region: "auto",
|
||||
rootPrefix: "obsync/v2",
|
||||
accountKey: "team",
|
||||
|
|
@ -263,6 +266,7 @@ function createPlugin(settings: Partial<TestSettings> = {}): TestPlugin {
|
|||
plugin.settings = {
|
||||
endpoint: "https://example.com",
|
||||
bucket: "vault",
|
||||
addressingStyle: "auto",
|
||||
region: "auto",
|
||||
accessKeyId: "key",
|
||||
secretAccessKey: "secret",
|
||||
|
|
@ -300,6 +304,7 @@ type TestPlugin = {
|
|||
interface TestSettings {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
addressingStyle: "auto" | "path" | "virtual-hosted";
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
|
|
|
|||
31
src/main.ts
31
src/main.ts
|
|
@ -4,12 +4,13 @@ import { createRandomId, createVaultId, normalizeKey } from "./core/ids";
|
|||
import SettingsApp from "./settings/SettingsApp.vue";
|
||||
import "./styles.scss";
|
||||
import { releaseDeletedContent, syncOnce, type LocalSyncState } from "./sync/engine";
|
||||
import { S3ObjectStore } from "./store/s3";
|
||||
import { S3ObjectStore, type S3AddressingStyle } from "./store/s3";
|
||||
import { ObsidianVaultIO } from "./vault-io";
|
||||
|
||||
interface ObsyncSettings {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
addressingStyle: S3AddressingStyle;
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
|
|
@ -29,6 +30,7 @@ interface ConnectionConfig {
|
|||
schemaVersion: 1;
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
addressingStyle?: S3AddressingStyle;
|
||||
region: string;
|
||||
rootPrefix: string;
|
||||
accountKey: string;
|
||||
|
|
@ -41,6 +43,7 @@ interface ConnectionConfig {
|
|||
const DEFAULT_SETTINGS: ObsyncSettings = {
|
||||
endpoint: "",
|
||||
bucket: "",
|
||||
addressingStyle: "auto",
|
||||
region: "",
|
||||
accessKeyId: "",
|
||||
secretAccessKey: "",
|
||||
|
|
@ -255,7 +258,7 @@ export default class ObsyncPlugin extends Plugin {
|
|||
|
||||
this.pruneLocalDeletedState();
|
||||
await this.saveSettings();
|
||||
this.showNotice(`Obsync 已释放：清理删除记录 ${result.deletedTombstones}，删除 Blob ${result.deletedBlobs}。`);
|
||||
this.showNotice(`Obsync 已释放：清理文件删除记录 ${result.deletedTombstones}，目录删除记录 ${result.deletedDirectoryTombstones}，删除 Blob ${result.deletedBlobs}。`);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -325,6 +328,7 @@ export default class ObsyncPlugin extends Plugin {
|
|||
return new S3ObjectStore({
|
||||
endpoint: this.settings.endpoint,
|
||||
bucket: this.settings.bucket,
|
||||
addressingStyle: this.settings.addressingStyle,
|
||||
region: this.settings.region || "auto",
|
||||
accessKeyId: this.settings.accessKeyId,
|
||||
secretAccessKey: this.settings.secretAccessKey,
|
||||
|
|
@ -358,6 +362,12 @@ export default class ObsyncPlugin extends Plugin {
|
|||
delete this.settings.syncState.files[path];
|
||||
}
|
||||
}
|
||||
|
||||
for (const [path, directoryState] of Object.entries(this.settings.syncState.directories ?? {})) {
|
||||
if (directoryState.deleted) {
|
||||
delete this.settings.syncState.directories?.[path];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async loadSettings(): Promise<void> {
|
||||
|
|
@ -408,6 +418,7 @@ export default class ObsyncPlugin extends Plugin {
|
|||
schemaVersion: 1,
|
||||
endpoint: this.settings.endpoint,
|
||||
bucket: this.settings.bucket,
|
||||
addressingStyle: this.settings.addressingStyle,
|
||||
region: this.settings.region,
|
||||
rootPrefix: this.settings.rootPrefix,
|
||||
accountKey: this.settings.accountKey,
|
||||
|
|
@ -429,6 +440,7 @@ export default class ObsyncPlugin extends Plugin {
|
|||
await this.updateSettings({
|
||||
endpoint: config.endpoint,
|
||||
bucket: config.bucket,
|
||||
addressingStyle: config.addressingStyle || "auto",
|
||||
region: config.region,
|
||||
rootPrefix: config.rootPrefix,
|
||||
accountKey: config.accountKey,
|
||||
|
|
@ -487,6 +499,7 @@ function parseConnectionConfig(configText: string): ConnectionConfig {
|
|||
schemaVersion: 1 as const,
|
||||
endpoint: readRequiredString(parsed, "endpoint"),
|
||||
bucket: readRequiredString(parsed, "bucket"),
|
||||
addressingStyle: readAddressingStyle(parsed, "addressingStyle", "auto"),
|
||||
region: readOptionalString(parsed, "region", "auto"),
|
||||
rootPrefix: readOptionalString(parsed, "rootPrefix", "obsync/v1"),
|
||||
accountKey: readOptionalString(parsed, "accountKey", "default"),
|
||||
|
|
@ -503,6 +516,20 @@ function parseConnectionConfig(configText: string): ConnectionConfig {
|
|||
return config;
|
||||
}
|
||||
|
||||
function readAddressingStyle(record: Record<string, unknown>, key: string, fallback: S3AddressingStyle): S3AddressingStyle {
|
||||
const value = record[key];
|
||||
|
||||
if (value === undefined || value === null || value === "") {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
if (value === "auto" || value === "path" || value === "virtual-hosted") {
|
||||
return value;
|
||||
}
|
||||
|
||||
throw new Error(`连接配置字段 ${key} 格式不正确。`);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { computed, ref, watchEffect } from "vue";
|
|||
type ObsyncSettings = {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
addressingStyle: "auto" | "path" | "virtual-hosted";
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
|
|
@ -97,6 +98,23 @@ watchEffect(() => {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="obsync-row">
|
||||
<div class="obsync-row-copy">
|
||||
<h3>地址风格</h3>
|
||||
<p>自动适配常见服务，也可手动指定请求地址格式。</p>
|
||||
</div>
|
||||
<div class="obsync-control">
|
||||
<select
|
||||
:value="props.settings.addressingStyle"
|
||||
@change="emit('update', { addressingStyle: ($event.target as HTMLSelectElement).value as ObsyncSettings['addressingStyle'] })"
|
||||
>
|
||||
<option value="auto">自动</option>
|
||||
<option value="path">路径风格</option>
|
||||
<option value="virtual-hosted">虚拟主机风格</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="obsync-row">
|
||||
<div class="obsync-row-copy">
|
||||
<h3>Region</h3>
|
||||
|
|
|
|||
|
|
@ -111,12 +111,76 @@ describe("S3ObjectStore", () => {
|
|||
expect(requests).toHaveLength(2);
|
||||
expect(requests[1]?.url).toContain("continuation-token=page+2");
|
||||
});
|
||||
|
||||
test("uses virtual-hosted style automatically for Aliyun OSS endpoints", async () => {
|
||||
const requests: HttpRequest[] = [];
|
||||
const store = createStore(async (request) => {
|
||||
requests.push(request);
|
||||
return {
|
||||
status: 200,
|
||||
text: "",
|
||||
arrayBuffer: new ArrayBuffer(0),
|
||||
};
|
||||
}, {
|
||||
endpoint: "https://oss-cn-beijing.aliyuncs.com",
|
||||
bucket: "cohere-test",
|
||||
region: "oss-cn-beijing",
|
||||
});
|
||||
|
||||
await store.writeObject("files/notes/today.md", new TextEncoder().encode("hello"));
|
||||
|
||||
expect(requests[0]?.url).toBe("https://cohere-test.oss-cn-beijing.aliyuncs.com/obsync/v1/vaults/vlt_TEST/files/notes/today.md");
|
||||
});
|
||||
|
||||
test("can force path style addressing", async () => {
|
||||
const requests: HttpRequest[] = [];
|
||||
const store = createStore(async (request) => {
|
||||
requests.push(request);
|
||||
return {
|
||||
status: 200,
|
||||
text: "",
|
||||
arrayBuffer: new ArrayBuffer(0),
|
||||
};
|
||||
}, {
|
||||
addressingStyle: "path",
|
||||
endpoint: "https://oss-cn-beijing.aliyuncs.com",
|
||||
bucket: "cohere-test",
|
||||
});
|
||||
|
||||
await store.writeObject("files/notes/today.md", new TextEncoder().encode("hello"));
|
||||
|
||||
expect(requests[0]?.url).toBe("https://oss-cn-beijing.aliyuncs.com/cohere-test/obsync/v1/vaults/vlt_TEST/files/notes/today.md");
|
||||
});
|
||||
|
||||
test("can force virtual-hosted style addressing", async () => {
|
||||
const requests: HttpRequest[] = [];
|
||||
const store = createStore(async (request) => {
|
||||
requests.push(request);
|
||||
return {
|
||||
status: 200,
|
||||
text: "",
|
||||
arrayBuffer: new ArrayBuffer(0),
|
||||
};
|
||||
}, {
|
||||
addressingStyle: "virtual-hosted",
|
||||
endpoint: "https://s3.example.com",
|
||||
bucket: "cohere-test",
|
||||
});
|
||||
|
||||
await store.writeObject("files/notes/today.md", new TextEncoder().encode("hello"));
|
||||
|
||||
expect(requests[0]?.url).toBe("https://cohere-test.s3.example.com/obsync/v1/vaults/vlt_TEST/files/notes/today.md");
|
||||
});
|
||||
});
|
||||
|
||||
function createStore(request: (request: HttpRequest) => Promise<{ status: number; text: string; arrayBuffer: ArrayBuffer }>) {
|
||||
function createStore(
|
||||
request: (request: HttpRequest) => Promise<{ status: number; text: string; arrayBuffer: ArrayBuffer }>,
|
||||
options: Partial<ConstructorParameters<typeof S3ObjectStore>[0]> = {},
|
||||
) {
|
||||
return new S3ObjectStore({
|
||||
endpoint: "https://s3.example.com",
|
||||
bucket: "my-bucket",
|
||||
addressingStyle: "auto",
|
||||
region: "auto",
|
||||
accessKeyId: "AKIA_TEST",
|
||||
secretAccessKey: "SECRET_TEST",
|
||||
|
|
@ -125,5 +189,6 @@ function createStore(request: (request: HttpRequest) => Promise<{ status: number
|
|||
deviceId: "dev_TEST",
|
||||
now: () => 1000,
|
||||
request,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ interface HttpResponse {
|
|||
interface S3ObjectStoreOptions {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
addressingStyle: S3AddressingStyle;
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
|
|
@ -27,6 +28,8 @@ interface S3ObjectStoreOptions {
|
|||
request(request: HttpRequest): Promise<HttpResponse>;
|
||||
}
|
||||
|
||||
export type S3AddressingStyle = "auto" | "path" | "virtual-hosted";
|
||||
|
||||
export class S3ObjectStore implements ObjectStore {
|
||||
private options: S3ObjectStoreOptions;
|
||||
private layout: ReturnType<typeof createRemoteLayout>;
|
||||
|
|
@ -127,7 +130,7 @@ export class S3ObjectStore implements ObjectStore {
|
|||
}
|
||||
|
||||
private async request(method: string, key: string, body?: Uint8Array, query?: Record<string, string>): Promise<HttpResponse> {
|
||||
const url = createObjectUrl(this.options.endpoint, this.options.bucket, key, query);
|
||||
const url = createObjectUrl(this.options.endpoint, this.options.bucket, key, this.options.addressingStyle, query);
|
||||
const bodyBytes = body ?? new Uint8Array();
|
||||
const headers = await createSignedHeaders({
|
||||
method,
|
||||
|
|
@ -148,13 +151,43 @@ export class S3ObjectStore implements ObjectStore {
|
|||
}
|
||||
}
|
||||
|
||||
function createObjectUrl(endpoint: string, bucket: string, key: string, query?: Record<string, string>): string {
|
||||
function createObjectUrl(endpoint: string, bucket: string, key: string, addressingStyle: S3AddressingStyle, query?: Record<string, string>): string {
|
||||
const base = endpoint.replace(/\/+$/g, "");
|
||||
const path = key ? `${encodePathPart(bucket)}/${encodeKey(key)}` : encodePathPart(bucket);
|
||||
const search = query ? `?${new URLSearchParams(query).toString()}` : "";
|
||||
|
||||
if (resolveAddressingStyle(endpoint, addressingStyle) === "virtual-hosted") {
|
||||
const url = new URL(base);
|
||||
url.hostname = `${bucket}.${url.hostname}`;
|
||||
const path = key ? encodeKey(key) : "";
|
||||
return `${url.origin}${url.pathname.replace(/\/+$/g, "")}/${path}${search}`;
|
||||
}
|
||||
|
||||
const path = key ? `${encodePathPart(bucket)}/${encodeKey(key)}` : encodePathPart(bucket);
|
||||
return `${base}/${path}${search}`;
|
||||
}
|
||||
|
||||
function resolveAddressingStyle(endpoint: string, addressingStyle: S3AddressingStyle): Exclude<S3AddressingStyle, "auto"> {
|
||||
if (addressingStyle !== "auto") {
|
||||
return addressingStyle;
|
||||
}
|
||||
|
||||
const hostname = safeHostname(endpoint);
|
||||
|
||||
if (hostname.endsWith(".aliyuncs.com") || hostname.endsWith(".myqcloud.com")) {
|
||||
return "virtual-hosted";
|
||||
}
|
||||
|
||||
return "path";
|
||||
}
|
||||
|
||||
function safeHostname(endpoint: string): string {
|
||||
try {
|
||||
return new URL(endpoint).hostname.toLowerCase();
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function encodeKey(key: string): string {
|
||||
return key.split("/").map(encodePathPart).join("/");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -125,14 +125,16 @@
|
|||
@apply w-80 shrink-0;
|
||||
}
|
||||
|
||||
.obsync-control input {
|
||||
.obsync-control input,
|
||||
.obsync-control select {
|
||||
@apply h-9 w-full min-w-0 rounded-md px-3 py-2 text-sm outline-none transition;
|
||||
background: var(--background-primary);
|
||||
border: 1px solid var(--background-modifier-border);
|
||||
color: var(--text-normal);
|
||||
}
|
||||
|
||||
.obsync-control input:focus {
|
||||
.obsync-control input:focus,
|
||||
.obsync-control select:focus {
|
||||
border-color: var(--interactive-accent);
|
||||
box-shadow: 0 0 0 2px var(--background-modifier-border-focus);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -644,6 +644,12 @@ describe("sync engine", () => {
|
|||
previousVersion: "ver_deleted",
|
||||
deletedBy: "dev_mac",
|
||||
};
|
||||
store.manifest.deletedDirectories["notes/empty"] = {
|
||||
deletedAt: 1000,
|
||||
deletedRevision: 3,
|
||||
previousVersion: "ver_empty",
|
||||
deletedBy: "dev_mac",
|
||||
};
|
||||
await store.putText(blobObjectKey(keptHash), "kept");
|
||||
await store.putText(blobObjectKey(deletedHash), "deleted");
|
||||
await store.putText(blobObjectKey(orphanHash), "orphan");
|
||||
|
|
@ -651,8 +657,10 @@ describe("sync engine", () => {
|
|||
const result = await releaseDeletedContent({ store, now: () => 5000 });
|
||||
|
||||
expect(result.deletedTombstones).toBe(1);
|
||||
expect(result.deletedDirectoryTombstones).toBe(1);
|
||||
expect(result.deletedBlobs).toBe(2);
|
||||
expect(store.manifest.deleted["notes/deleted.md"]).toBeUndefined();
|
||||
expect(store.manifest.deletedDirectories["notes/empty"]).toBeUndefined();
|
||||
expect(store.objects[blobObjectKey(keptHash)]).toBeDefined();
|
||||
expect(store.objects[blobObjectKey(deletedHash)]).toBeUndefined();
|
||||
expect(store.objects[blobObjectKey(orphanHash)]).toBeUndefined();
|
||||
|
|
@ -669,14 +677,22 @@ describe("sync engine", () => {
|
|||
previousVersion: "ver_deleted",
|
||||
deletedBy: "dev_mac",
|
||||
};
|
||||
store.manifest.deletedDirectories["notes/empty"] = {
|
||||
deletedAt: 1000,
|
||||
deletedRevision: 3,
|
||||
previousVersion: "ver_empty",
|
||||
deletedBy: "dev_mac",
|
||||
};
|
||||
await store.putText(blobObjectKey(deletedHash), "deleted");
|
||||
|
||||
const result = await releaseDeletedContent({ store, now: () => 5000 });
|
||||
|
||||
expect(result.locked).toBe(true);
|
||||
expect(result.deletedTombstones).toBe(0);
|
||||
expect(result.deletedDirectoryTombstones).toBe(0);
|
||||
expect(result.deletedBlobs).toBe(0);
|
||||
expect(store.manifest.deleted["notes/deleted.md"]).toBeDefined();
|
||||
expect(store.manifest.deletedDirectories["notes/empty"]).toBeDefined();
|
||||
expect(store.objects[blobObjectKey(deletedHash)]).toBeDefined();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -256,6 +256,7 @@ async function applyConflict(item: PlannedFile, input: SyncOnceInput, result: Sy
|
|||
|
||||
export interface ReleaseDeletedContentResult {
|
||||
deletedTombstones: number;
|
||||
deletedDirectoryTombstones: number;
|
||||
deletedBlobs: number;
|
||||
locked: boolean;
|
||||
}
|
||||
|
|
@ -264,14 +265,16 @@ export async function releaseDeletedContent(input: { store: ObjectStore; now():
|
|||
const locked = await input.store.acquireLock();
|
||||
|
||||
if (!locked) {
|
||||
return { deletedTombstones: 0, deletedBlobs: 0, locked: true };
|
||||
return { deletedTombstones: 0, deletedDirectoryTombstones: 0, deletedBlobs: 0, locked: true };
|
||||
}
|
||||
|
||||
try {
|
||||
const manifest = normalizeManifest(await input.store.readManifest());
|
||||
const deletedTombstones = Object.keys(manifest.deleted).length;
|
||||
const deletedDirectoryTombstones = Object.keys(manifest.deletedDirectories).length;
|
||||
|
||||
manifest.deleted = {};
|
||||
manifest.deletedDirectories = {};
|
||||
const referencedHashes = new Set(Object.values(manifest.paths).map((entry) => entry.contentHash));
|
||||
const knownBlobKeys = new Set(Object.values(manifest.blobs).map((entry) => entry.key));
|
||||
const listedBlobKeys = input.store.listObjectKeys ? await input.store.listObjectKeys("blobs/sha256/") : [];
|
||||
|
|
@ -291,7 +294,7 @@ export async function releaseDeletedContent(input: { store: ObjectStore; now():
|
|||
manifest.updatedAt = input.now();
|
||||
await input.store.writeManifest(manifest);
|
||||
|
||||
return { deletedTombstones, deletedBlobs, locked: false };
|
||||
return { deletedTombstones, deletedDirectoryTombstones, deletedBlobs, locked: false };
|
||||
} finally {
|
||||
await input.store.releaseLock();
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue