mirror of
https://github.com/panatgithub/AnkiHeadingSync.git
synced 2026-07-22 06:51:43 +00:00
fix: tighten module 3 indexing identity and caching
This commit is contained in:
parent
d3d356fb10
commit
a0ac404712
6 changed files with 224 additions and 47 deletions
|
|
@ -24,9 +24,10 @@
|
|||
|
||||
## 4. pending write-back 恢复策略
|
||||
|
||||
- 计划要求 pending write-back 可恢复,但 marker 缺失时无法仅靠 marker 重新定位
|
||||
- 因此增加一个严格受限的内部恢复规则:仅允许在“同一文件 + 相同 `rawBlockHash`”时复用既有 `cardId/noteId`
|
||||
- 这不是通用无 marker 身份推断,只用于本地已知 pending 或已存在状态的恢复
|
||||
- 模块 3 的正式身份只认 marker 中的 `cardId`
|
||||
- 若 marker 丢失,即视为新卡片块,重新生成新的 `cardId`
|
||||
- pending write-back 仅在 marker 仍保留 `cardId` 时恢复对应 `noteId`
|
||||
- 不按 `rawBlockHash` 对无 marker 块做长期身份复绑
|
||||
|
||||
## 5. 渲染策略
|
||||
|
||||
|
|
@ -60,7 +61,7 @@
|
|||
- 它会:
|
||||
- 全量重建文件状态与卡片状态
|
||||
- 为缺失 `cardId` 的块补齐 marker
|
||||
- 尝试用本地状态恢复已知 `noteId`
|
||||
- 仅对仍保留 `cardId` 的块恢复本地已知 `noteId`
|
||||
|
||||
## 9. 命令与提示
|
||||
|
||||
|
|
|
|||
105
src/application/services/AnkiBatchExecutor.test.ts
Normal file
105
src/application/services/AnkiBatchExecutor.test.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { NoteModelDetails } from "@/application/dto/NoteModelDetails";
|
||||
import type { IndexedCard } from "@/domain/manual-sync/entities/IndexedCard";
|
||||
import type { RenderedSyncCard } from "@/domain/manual-sync/entities/RenderedSyncCard";
|
||||
import type { ManualSyncPlan, PlannedCard } from "@/domain/manual-sync/value-objects/ManualSyncPlan";
|
||||
import { createModule3Settings, FakeManualSyncAnkiGateway } from "@/test-support/manualSyncFakes";
|
||||
|
||||
import { AnkiBatchExecutor } from "./AnkiBatchExecutor";
|
||||
|
||||
class CountingAnkiGateway extends FakeManualSyncAnkiGateway {
|
||||
public getModelDetailsCalls: string[] = [];
|
||||
|
||||
override async getModelDetails(modelName: string): Promise<NoteModelDetails> {
|
||||
this.getModelDetailsCalls.push(modelName);
|
||||
return super.getModelDetails(modelName);
|
||||
}
|
||||
}
|
||||
|
||||
describe("AnkiBatchExecutor", () => {
|
||||
it("caches note model details within a single execute run", async () => {
|
||||
const ankiGateway = new CountingAnkiGateway();
|
||||
ankiGateway.noteSummariesById.set(300, {
|
||||
noteId: 300,
|
||||
modelName: "Basic",
|
||||
cardIds: [700],
|
||||
});
|
||||
|
||||
const executor = new AnkiBatchExecutor(ankiGateway);
|
||||
const createOne = createPlannedCard("ahs_1");
|
||||
const createTwo = createPlannedCard("ahs_2");
|
||||
const updateOne = createPlannedCard("ahs_3", 300);
|
||||
const renderedCards = new Map<string, RenderedSyncCard>([
|
||||
[createOne.card.cardId, createRenderedSyncCard(createOne)],
|
||||
[createTwo.card.cardId, createRenderedSyncCard(createTwo)],
|
||||
[updateOne.card.cardId, createRenderedSyncCard(updateOne)],
|
||||
]);
|
||||
const plan: ManualSyncPlan = {
|
||||
toCreate: [createOne, createTwo],
|
||||
toUpdate: [updateOne],
|
||||
toRewriteMarker: [],
|
||||
toOrphan: [],
|
||||
unchangedCards: 0,
|
||||
};
|
||||
|
||||
const result = await executor.execute(
|
||||
plan,
|
||||
renderedCards,
|
||||
async (plannedCard) => createRenderedSyncCard(plannedCard),
|
||||
createModule3Settings().noteFieldMappings,
|
||||
);
|
||||
|
||||
expect(result.created).toBe(2);
|
||||
expect(result.updated).toBe(1);
|
||||
expect(ankiGateway.getModelDetailsCalls).toEqual(["Basic"]);
|
||||
});
|
||||
});
|
||||
|
||||
function createPlannedCard(cardId: string, noteId?: number): PlannedCard {
|
||||
return {
|
||||
card: createIndexedCard(cardId, noteId),
|
||||
noteId,
|
||||
deck: "Obsidian",
|
||||
noteModel: "Basic",
|
||||
renderConfigHash: "render-config",
|
||||
};
|
||||
}
|
||||
|
||||
function createRenderedSyncCard(plannedCard: PlannedCard): RenderedSyncCard {
|
||||
return {
|
||||
card: plannedCard.card,
|
||||
noteId: plannedCard.noteId,
|
||||
deck: plannedCard.deck,
|
||||
noteModel: plannedCard.noteModel,
|
||||
renderConfigHash: plannedCard.renderConfigHash,
|
||||
renderedFields: {
|
||||
title: plannedCard.card.heading,
|
||||
body: plannedCard.card.bodyMarkdown,
|
||||
},
|
||||
media: [],
|
||||
};
|
||||
}
|
||||
|
||||
function createIndexedCard(cardId: string, noteId?: number): IndexedCard {
|
||||
return {
|
||||
cardId,
|
||||
noteId,
|
||||
markerNoteId: noteId,
|
||||
filePath: "notes/example.md",
|
||||
cardType: "basic",
|
||||
heading: `Heading ${cardId}`,
|
||||
headingLevel: 4,
|
||||
bodyMarkdown: `Body ${cardId}`,
|
||||
blockStartOffset: 0,
|
||||
blockEndOffset: 10,
|
||||
blockStartLine: 1,
|
||||
bodyStartLine: 2,
|
||||
blockEndLine: 2,
|
||||
contentEndLine: 2,
|
||||
rawBlockText: `#### Heading ${cardId}\nBody ${cardId}`,
|
||||
rawBlockHash: `hash-${cardId}`,
|
||||
tagsHint: [],
|
||||
markerState: noteId ? "card-and-note" : "card-only",
|
||||
};
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ export class AnkiBatchExecutor {
|
|||
renderOnDemand: (plannedCard: PlannedCard) => Promise<RenderedSyncCard>,
|
||||
noteFieldMappings: Record<string, NoteModelFieldMapping>,
|
||||
): Promise<AnkiBatchExecutionResult> {
|
||||
const modelDetailsCache = new Map<string, Promise<Awaited<ReturnType<AnkiGateway["getModelDetails"]>>>>();
|
||||
const resolvedNoteIds = new Map<string, number | undefined>();
|
||||
const touchedCardIds = new Set<string>();
|
||||
const markerWriteMap = new Map<string, PlannedCard>();
|
||||
|
|
@ -84,7 +85,7 @@ export class AnkiBatchExecutor {
|
|||
await Promise.all(batch.map(async (renderedCard) => ({
|
||||
deckName: renderedCard.deck,
|
||||
modelName: renderedCard.noteModel,
|
||||
fields: await this.mapFields(renderedCard, noteFieldMappings),
|
||||
fields: await this.mapFields(renderedCard, noteFieldMappings, modelDetailsCache),
|
||||
tags: renderedCard.card.tagsHint,
|
||||
}))),
|
||||
);
|
||||
|
|
@ -114,7 +115,7 @@ export class AnkiBatchExecutor {
|
|||
await Promise.all(batch.map(async ({ plannedCard, renderedCard }) => ({
|
||||
noteId: plannedCard.noteId ?? 0,
|
||||
deckName: renderedCard.deck,
|
||||
fields: await this.mapFields(renderedCard, noteFieldMappings),
|
||||
fields: await this.mapFields(renderedCard, noteFieldMappings, modelDetailsCache),
|
||||
}))),
|
||||
);
|
||||
});
|
||||
|
|
@ -182,8 +183,9 @@ export class AnkiBatchExecutor {
|
|||
private async mapFields(
|
||||
renderedCard: RenderedSyncCard,
|
||||
noteFieldMappings: Record<string, NoteModelFieldMapping>,
|
||||
modelDetailsCache: Map<string, Promise<Awaited<ReturnType<AnkiGateway["getModelDetails"]>>>>,
|
||||
): Promise<Record<string, string>> {
|
||||
const modelDetails = await this.ankiGateway.getModelDetails(renderedCard.noteModel);
|
||||
const modelDetails = await this.getModelDetails(renderedCard.noteModel, modelDetailsCache);
|
||||
return this.noteFieldMappingService.mapRenderedCard(
|
||||
{
|
||||
type: renderedCard.card.cardType,
|
||||
|
|
@ -195,6 +197,24 @@ export class AnkiBatchExecutor {
|
|||
);
|
||||
}
|
||||
|
||||
private async getModelDetails(
|
||||
noteModel: string,
|
||||
modelDetailsCache: Map<string, Promise<Awaited<ReturnType<AnkiGateway["getModelDetails"]>>>>,
|
||||
): Promise<Awaited<ReturnType<AnkiGateway["getModelDetails"]>>> {
|
||||
const cached = modelDetailsCache.get(noteModel);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const pending = this.ankiGateway.getModelDetails(noteModel).catch((error) => {
|
||||
modelDetailsCache.delete(noteModel);
|
||||
throw error;
|
||||
});
|
||||
|
||||
modelDetailsCache.set(noteModel, pending);
|
||||
return pending;
|
||||
}
|
||||
|
||||
private async uploadMedia(renderedCards: RenderedSyncCard[]): Promise<number> {
|
||||
const uniqueMedia = new Map<string, MediaAsset>();
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import type { ManualSyncVaultGateway } from "@/application/ports/ManualSyncVault
|
|||
import { ScanScopeService } from "@/application/services/ScanScopeService";
|
||||
import type { IndexedCard } from "@/domain/manual-sync/entities/IndexedCard";
|
||||
import type { IndexedFile } from "@/domain/manual-sync/entities/IndexedFile";
|
||||
import type { CardState, PluginState } from "@/domain/manual-sync/entities/PluginState";
|
||||
import type { CardState, PendingWriteBackState, PluginState } from "@/domain/manual-sync/entities/PluginState";
|
||||
import { CardIndexingService } from "@/domain/manual-sync/services/CardIndexingService";
|
||||
|
||||
export interface FileIndexerResult {
|
||||
|
|
@ -15,6 +15,11 @@ export interface FileIndexerResult {
|
|||
skippedUnchangedCards: number;
|
||||
}
|
||||
|
||||
interface StateIndex {
|
||||
cardsByFilePath: Map<string, CardState[]>;
|
||||
pendingByFilePath: Map<string, PendingWriteBackState[]>;
|
||||
}
|
||||
|
||||
export class FileIndexerService {
|
||||
constructor(
|
||||
private readonly vaultGateway: ManualSyncVaultGateway,
|
||||
|
|
@ -24,7 +29,7 @@ export class FileIndexerService {
|
|||
|
||||
async indexVault(settings: PluginSettings, state: PluginState, forceReadAll = false): Promise<FileIndexerResult> {
|
||||
const refs = this.scanScopeService.filter(await this.vaultGateway.listMarkdownFileRefs(), settings.includeFolders, settings.excludeFolders);
|
||||
return this.indexRefs(refs, settings, state, forceReadAll);
|
||||
return this.indexRefs(refs, settings, state, this.buildStateIndex(state), forceReadAll);
|
||||
}
|
||||
|
||||
async indexFile(filePath: string, settings: PluginSettings, state: PluginState): Promise<FileIndexerResult> {
|
||||
|
|
@ -37,12 +42,13 @@ export class FileIndexerService {
|
|||
}
|
||||
|
||||
const fileStamp = reference ? createFileStamp(reference.mtime, reference.size) : `${Date.now()}:${sourceFile.content.length}`;
|
||||
const stateIndex = this.buildStateIndex(state);
|
||||
const indexedFile = this.cardIndexingService.index(sourceFile, {
|
||||
qaHeadingLevel: settings.qaHeadingLevel,
|
||||
clozeHeadingLevel: settings.clozeHeadingLevel,
|
||||
fileStamp,
|
||||
knownCards: Object.values(state.cards).filter((card) => card.filePath === filePath),
|
||||
pendingWriteBack: state.pendingWriteBack.filter((pending) => pending.filePath === filePath),
|
||||
knownCards: stateIndex.cardsByFilePath.get(filePath) ?? [],
|
||||
pendingWriteBack: stateIndex.pendingByFilePath.get(filePath) ?? [],
|
||||
});
|
||||
|
||||
return {
|
||||
|
|
@ -59,6 +65,7 @@ export class FileIndexerService {
|
|||
refs: Awaited<ReturnType<ManualSyncVaultGateway["listMarkdownFileRefs"]>>,
|
||||
settings: PluginSettings,
|
||||
state: PluginState,
|
||||
stateIndex: StateIndex,
|
||||
forceReadAll: boolean,
|
||||
): Promise<FileIndexerResult> {
|
||||
const indexedFiles: IndexedFile[] = [];
|
||||
|
|
@ -69,8 +76,9 @@ export class FileIndexerService {
|
|||
for (const ref of refs) {
|
||||
const fileStamp = createFileStamp(ref.mtime, ref.size);
|
||||
const existingFileState = state.files[ref.path];
|
||||
const hasPendingWriteBack = state.pendingWriteBack.some((pending) => pending.filePath === ref.path);
|
||||
const knownCards = Object.values(state.cards).filter((card) => card.filePath === ref.path);
|
||||
const pendingWriteBack = stateIndex.pendingByFilePath.get(ref.path) ?? [];
|
||||
const hasPendingWriteBack = pendingWriteBack.length > 0;
|
||||
const knownCards = stateIndex.cardsByFilePath.get(ref.path) ?? [];
|
||||
const hasMissingKnownCard = existingFileState?.cardIds.some((cardId) => !state.cards[cardId]) ?? false;
|
||||
const shouldRead =
|
||||
forceReadAll ||
|
||||
|
|
@ -107,7 +115,7 @@ export class FileIndexerService {
|
|||
clozeHeadingLevel: settings.clozeHeadingLevel,
|
||||
fileStamp,
|
||||
knownCards,
|
||||
pendingWriteBack: state.pendingWriteBack.filter((pending) => pending.filePath === ref.path),
|
||||
pendingWriteBack,
|
||||
});
|
||||
|
||||
indexedFiles.push(indexedFile);
|
||||
|
|
@ -123,6 +131,36 @@ export class FileIndexerService {
|
|||
skippedUnchangedCards,
|
||||
};
|
||||
}
|
||||
|
||||
private buildStateIndex(state: PluginState): StateIndex {
|
||||
const cardsByFilePath = new Map<string, CardState[]>();
|
||||
const pendingByFilePath = new Map<string, PendingWriteBackState[]>();
|
||||
|
||||
for (const card of Object.values(state.cards)) {
|
||||
const cards = cardsByFilePath.get(card.filePath);
|
||||
if (cards) {
|
||||
cards.push(card);
|
||||
continue;
|
||||
}
|
||||
|
||||
cardsByFilePath.set(card.filePath, [card]);
|
||||
}
|
||||
|
||||
for (const pending of state.pendingWriteBack) {
|
||||
const pendings = pendingByFilePath.get(pending.filePath);
|
||||
if (pendings) {
|
||||
pendings.push(pending);
|
||||
continue;
|
||||
}
|
||||
|
||||
pendingByFilePath.set(pending.filePath, [pending]);
|
||||
}
|
||||
|
||||
return {
|
||||
cardsByFilePath,
|
||||
pendingByFilePath,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function restoreIndexedCard(card: CardState): IndexedCard {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { hashString } from "@/domain/shared/hash";
|
||||
|
||||
import { CardIndexingService } from "./CardIndexingService";
|
||||
|
||||
describe("CardIndexingService", () => {
|
||||
|
|
@ -28,7 +26,7 @@ describe("CardIndexingService", () => {
|
|||
expect(indexedFile.cards[0]?.bodyMarkdown).toBe("Answer");
|
||||
});
|
||||
|
||||
it("reuses cardId and noteId from pending write-back when marker is still missing", () => {
|
||||
it("treats a missing marker as a new card even if local pending state has the same raw block hash", () => {
|
||||
const service = new CardIndexingService();
|
||||
const indexedFile = service.index(
|
||||
{
|
||||
|
|
@ -48,12 +46,56 @@ describe("CardIndexingService", () => {
|
|||
noteId: 42,
|
||||
expectedFileHash: "hash",
|
||||
targetMarker: "<!-- AHS:card=ahs_known note=42 -->",
|
||||
rawBlockHash: hashString(["#### Prompt", "Answer"].join("\n")),
|
||||
rawBlockHash: "hash-card",
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(indexedFile.cards[0]?.cardId).not.toBe("ahs_known");
|
||||
expect(indexedFile.cards[0]?.noteId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("restores noteId when the marker still carries cardId", () => {
|
||||
const service = new CardIndexingService();
|
||||
const indexedFile = service.index(
|
||||
{
|
||||
path: "notes/example.md",
|
||||
basename: "example",
|
||||
content: ["#### Prompt", "Answer", "<!-- AHS:card=ahs_known -->"].join("\n"),
|
||||
},
|
||||
{
|
||||
qaHeadingLevel: 4,
|
||||
clozeHeadingLevel: 5,
|
||||
fileStamp: "1:1",
|
||||
knownCards: [
|
||||
{
|
||||
cardId: "ahs_known",
|
||||
noteId: 42,
|
||||
filePath: "notes/example.md",
|
||||
heading: "Prompt",
|
||||
headingLevel: 4,
|
||||
bodyMarkdown: "Answer",
|
||||
cardType: "basic",
|
||||
blockStartOffset: 0,
|
||||
blockEndOffset: 13,
|
||||
blockStartLine: 1,
|
||||
bodyStartLine: 2,
|
||||
blockEndLine: 3,
|
||||
contentEndLine: 2,
|
||||
rawBlockText: ["#### Prompt", "Answer"].join("\n"),
|
||||
rawBlockHash: "hash-card",
|
||||
renderConfigHash: "render-hash",
|
||||
deck: "Obsidian",
|
||||
tagsHint: [],
|
||||
lastSyncedAt: 1,
|
||||
orphan: false,
|
||||
},
|
||||
],
|
||||
pendingWriteBack: [],
|
||||
},
|
||||
);
|
||||
|
||||
expect(indexedFile.cards[0]).toMatchObject({
|
||||
cardId: "ahs_known",
|
||||
noteId: 42,
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ export class CardIndexingService {
|
|||
const cards: IndexedCard[] = [];
|
||||
const knownCardsById = new Map(context.knownCards.map((card) => [card.cardId, card]));
|
||||
const pendingByCardId = new Map(context.pendingWriteBack.map((pending) => [pending.cardId, pending]));
|
||||
const reusableKnownCards = context.knownCards.filter((card) => !card.orphan);
|
||||
const usedCardIds = new Set<string>();
|
||||
|
||||
for (let headingIndex = 0; headingIndex < headings.length; headingIndex += 1) {
|
||||
|
|
@ -63,14 +62,10 @@ export class CardIndexingService {
|
|||
const rawBlockText = [lines[heading.lineIndex], ...trimmedBodyLines].join("\n").trimEnd();
|
||||
const rawBlockHash = hashString(rawBlockText);
|
||||
const resolvedIdentity = this.resolveIdentity(
|
||||
sourceFile.path,
|
||||
rawBlockHash,
|
||||
marker.markerCardId,
|
||||
marker.markerNoteId,
|
||||
knownCardsById,
|
||||
pendingByCardId,
|
||||
reusableKnownCards,
|
||||
usedCardIds,
|
||||
);
|
||||
|
||||
usedCardIds.add(resolvedIdentity.cardId);
|
||||
|
|
@ -110,14 +105,10 @@ export class CardIndexingService {
|
|||
}
|
||||
|
||||
private resolveIdentity(
|
||||
filePath: string,
|
||||
rawBlockHash: string,
|
||||
markerCardId: string | undefined,
|
||||
markerNoteId: number | undefined,
|
||||
knownCardsById: Map<string, CardState>,
|
||||
pendingByCardId: Map<string, PendingWriteBackState>,
|
||||
reusableKnownCards: CardState[],
|
||||
usedCardIds: Set<string>,
|
||||
): { cardId: string; noteId?: number } {
|
||||
if (markerCardId) {
|
||||
const pending = pendingByCardId.get(markerCardId);
|
||||
|
|
@ -129,26 +120,6 @@ export class CardIndexingService {
|
|||
};
|
||||
}
|
||||
|
||||
const pendingMatch = Array.from(pendingByCardId.values()).find(
|
||||
(pending) => pending.filePath === filePath && pending.rawBlockHash === rawBlockHash && !usedCardIds.has(pending.cardId),
|
||||
);
|
||||
if (pendingMatch) {
|
||||
return {
|
||||
cardId: pendingMatch.cardId,
|
||||
noteId: pendingMatch.noteId,
|
||||
};
|
||||
}
|
||||
|
||||
const knownMatch = reusableKnownCards.find(
|
||||
(card) => card.filePath === filePath && card.rawBlockHash === rawBlockHash && !usedCardIds.has(card.cardId),
|
||||
);
|
||||
if (knownMatch) {
|
||||
return {
|
||||
cardId: knownMatch.cardId,
|
||||
noteId: knownMatch.noteId,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
cardId: this.markerService.generateCardId(),
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue