mirror of
https://github.com/taskgenius/taskgenius-plugin.git
synced 2026-07-22 06:40:25 +00:00
fix: better performance
This commit is contained in:
parent
b0868745b3
commit
024fdc7dff
8 changed files with 725 additions and 29 deletions
|
|
@ -25,6 +25,8 @@ describe("FileMetadataTaskParser", () => {
|
|||
taskContentFromMetadata: "title",
|
||||
defaultTaskStatus: " ",
|
||||
enableWorkerProcessing: true,
|
||||
enableMtimeOptimization: false,
|
||||
mtimeCacheSize: 1000,
|
||||
};
|
||||
parser = new FileMetadataTaskParser(config);
|
||||
});
|
||||
|
|
@ -206,6 +208,8 @@ describe("FileMetadataTaskParser", () => {
|
|||
taskContentFromMetadata: "title",
|
||||
defaultTaskStatus: " ",
|
||||
enableWorkerProcessing: true,
|
||||
enableMtimeOptimization: false,
|
||||
mtimeCacheSize: 1000,
|
||||
};
|
||||
const disabledParser = new FileMetadataTaskParser(disabledConfig);
|
||||
|
||||
|
|
@ -358,6 +362,8 @@ describe("FileMetadataTaskUpdater", () => {
|
|||
taskContentFromMetadata: "title",
|
||||
defaultTaskStatus: " ",
|
||||
enableWorkerProcessing: true,
|
||||
enableMtimeOptimization: false,
|
||||
mtimeCacheSize: 1000,
|
||||
};
|
||||
|
||||
// Mock Obsidian App
|
||||
|
|
|
|||
172
src/__tests__/TaskIndexer.mtime.test.ts
Normal file
172
src/__tests__/TaskIndexer.mtime.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/**
|
||||
* Tests for TaskIndexer mtime-based caching functionality
|
||||
*/
|
||||
|
||||
import { TaskIndexer } from "../utils/import/TaskIndexer";
|
||||
import { Task } from "../types/task";
|
||||
|
||||
// Mock obsidian Component class
|
||||
jest.mock("obsidian", () => ({
|
||||
...jest.requireActual("obsidian"),
|
||||
Component: class {
|
||||
registerEvent = jest.fn();
|
||||
unload = jest.fn();
|
||||
},
|
||||
TFile: jest.fn(),
|
||||
}));
|
||||
|
||||
// Mock dependencies
|
||||
const mockApp = {} as any;
|
||||
const mockVault = {
|
||||
on: jest.fn().mockReturnValue({}),
|
||||
off: jest.fn(),
|
||||
} as any;
|
||||
const mockMetadataCache = {} as any;
|
||||
|
||||
describe("TaskIndexer mtime functionality", () => {
|
||||
let indexer: TaskIndexer;
|
||||
|
||||
beforeEach(() => {
|
||||
indexer = new TaskIndexer(mockApp, mockVault, mockMetadataCache);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (indexer && typeof indexer.unload === 'function') {
|
||||
indexer.unload();
|
||||
}
|
||||
});
|
||||
|
||||
describe("mtime comparison", () => {
|
||||
test("should detect file changes when mtime is newer", () => {
|
||||
const filePath = "test.md";
|
||||
const oldMtime = 1000;
|
||||
const newMtime = 2000;
|
||||
|
||||
// Set initial mtime
|
||||
indexer.updateFileMtime(filePath, oldMtime);
|
||||
|
||||
// Check if file is changed with newer mtime
|
||||
expect(indexer.isFileChanged(filePath, newMtime)).toBe(true);
|
||||
});
|
||||
|
||||
test("should not detect changes when mtime is same", () => {
|
||||
const filePath = "test.md";
|
||||
const mtime = 1000;
|
||||
|
||||
// Set initial mtime
|
||||
indexer.updateFileMtime(filePath, mtime);
|
||||
|
||||
// Check if file is changed with same mtime
|
||||
expect(indexer.isFileChanged(filePath, mtime)).toBe(false);
|
||||
});
|
||||
|
||||
test("should detect changes for unknown files", () => {
|
||||
const filePath = "unknown.md";
|
||||
const mtime = 1000;
|
||||
|
||||
// Check if unknown file is considered changed
|
||||
expect(indexer.isFileChanged(filePath, mtime)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("cache validation", () => {
|
||||
test("should have valid cache when file hasn't changed and has tasks", () => {
|
||||
const filePath = "test.md";
|
||||
const mtime = 1000;
|
||||
const tasks: Task[] = [
|
||||
{
|
||||
id: "task1",
|
||||
content: "Test task",
|
||||
filePath,
|
||||
line: 1,
|
||||
completed: false,
|
||||
status: " ",
|
||||
originalMarkdown: "- [ ] Test task",
|
||||
metadata: {
|
||||
tags: [],
|
||||
project: undefined,
|
||||
context: undefined,
|
||||
priority: undefined,
|
||||
dueDate: undefined,
|
||||
startDate: undefined,
|
||||
scheduledDate: undefined,
|
||||
completedDate: undefined,
|
||||
cancelledDate: undefined,
|
||||
createdDate: undefined,
|
||||
recurrence: undefined,
|
||||
dependsOn: [],
|
||||
onCompletion: undefined,
|
||||
taskId: undefined,
|
||||
children: [],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// Add tasks and set mtime
|
||||
indexer.updateIndexWithTasks(filePath, tasks, mtime);
|
||||
|
||||
// Check if cache is valid
|
||||
expect(indexer.hasValidCache(filePath, mtime)).toBe(true);
|
||||
});
|
||||
|
||||
test("should not have valid cache when file has changed", () => {
|
||||
const filePath = "test.md";
|
||||
const oldMtime = 1000;
|
||||
const newMtime = 2000;
|
||||
const tasks: Task[] = [];
|
||||
|
||||
// Add tasks with old mtime
|
||||
indexer.updateIndexWithTasks(filePath, tasks, oldMtime);
|
||||
|
||||
// Check if cache is invalid with new mtime
|
||||
expect(indexer.hasValidCache(filePath, newMtime)).toBe(false);
|
||||
});
|
||||
|
||||
test("should not have valid cache when no tasks exist", () => {
|
||||
const filePath = "test.md";
|
||||
const mtime = 1000;
|
||||
|
||||
// Don't add any tasks, just set mtime
|
||||
indexer.updateFileMtime(filePath, mtime);
|
||||
|
||||
// Check if cache is invalid when no tasks exist
|
||||
expect(indexer.hasValidCache(filePath, mtime)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("cache cleanup", () => {
|
||||
test("should clean up file cache properly", () => {
|
||||
const filePath = "test.md";
|
||||
const mtime = 1000;
|
||||
const tasks: Task[] = [];
|
||||
|
||||
// Add tasks and set mtime
|
||||
indexer.updateIndexWithTasks(filePath, tasks, mtime);
|
||||
|
||||
// Verify cache exists
|
||||
expect(indexer.getFileLastMtime(filePath)).toBe(mtime);
|
||||
|
||||
// Clean up cache
|
||||
indexer.cleanupFileCache(filePath);
|
||||
|
||||
// Verify cache is cleaned
|
||||
expect(indexer.getFileLastMtime(filePath)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("cache consistency", () => {
|
||||
test("should validate and fix cache consistency", () => {
|
||||
const filePath = "test.md";
|
||||
const mtime = 1000;
|
||||
|
||||
// Manually add mtime without tasks (inconsistent state)
|
||||
indexer.updateFileMtime(filePath, mtime);
|
||||
|
||||
// Validate consistency (should clean up orphaned mtime)
|
||||
indexer.validateCacheConsistency();
|
||||
|
||||
// Verify orphaned mtime is cleaned up
|
||||
expect(indexer.getFileLastMtime(filePath)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
257
src/__tests__/TaskWorkerManager.mtime.test.ts
Normal file
257
src/__tests__/TaskWorkerManager.mtime.test.ts
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
/**
|
||||
* Integration tests for TaskWorkerManager mtime optimization
|
||||
*/
|
||||
|
||||
import { TaskWorkerManager } from "../utils/workers/TaskWorkerManager";
|
||||
import { TaskIndexer } from "../utils/import/TaskIndexer";
|
||||
import { TFile } from "obsidian";
|
||||
|
||||
// Mock dependencies
|
||||
const mockVault = {
|
||||
cachedRead: jest.fn(),
|
||||
} as any;
|
||||
|
||||
const mockMetadataCache = {
|
||||
getFileCache: jest.fn(),
|
||||
} as any;
|
||||
|
||||
const mockApp = {} as any;
|
||||
|
||||
// Mock TFile
|
||||
const createMockFile = (path: string, mtime: number): TFile => ({
|
||||
path,
|
||||
stat: { mtime, ctime: mtime, size: 100 },
|
||||
extension: "md",
|
||||
name: path.split("/").pop() || path,
|
||||
basename: path.split("/").pop()?.replace(".md", "") || path,
|
||||
} as any);
|
||||
|
||||
describe("TaskWorkerManager mtime optimization", () => {
|
||||
let workerManager: TaskWorkerManager;
|
||||
let indexer: TaskIndexer;
|
||||
|
||||
beforeEach(() => {
|
||||
// Create indexer
|
||||
indexer = new TaskIndexer(mockApp, mockVault, mockMetadataCache);
|
||||
|
||||
// Create worker manager with mtime optimization enabled
|
||||
workerManager = new TaskWorkerManager(mockVault, mockMetadataCache, {
|
||||
maxWorkers: 1,
|
||||
settings: {
|
||||
fileParsingConfig: {
|
||||
enableMtimeOptimization: true,
|
||||
mtimeCacheSize: 1000,
|
||||
enableFileMetadataParsing: false,
|
||||
metadataFieldsToParseAsTasks: [],
|
||||
enableTagBasedTaskParsing: false,
|
||||
tagsToParseAsTasks: [],
|
||||
taskContentFromMetadata: "title",
|
||||
defaultTaskStatus: " ",
|
||||
enableWorkerProcessing: true,
|
||||
},
|
||||
preferMetadataFormat: "tasks",
|
||||
useDailyNotePathAsDate: false,
|
||||
dailyNoteFormat: "yyyy-MM-dd",
|
||||
useAsDateType: "due",
|
||||
dailyNotePath: "",
|
||||
ignoreHeading: "",
|
||||
focusHeading: "",
|
||||
fileMetadataInheritance: undefined,
|
||||
},
|
||||
});
|
||||
|
||||
// Set indexer reference
|
||||
workerManager.setTaskIndexer(indexer);
|
||||
|
||||
// Mock vault.cachedRead to return empty content
|
||||
mockVault.cachedRead.mockResolvedValue("");
|
||||
mockMetadataCache.getFileCache.mockReturnValue(null);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
workerManager.unload();
|
||||
indexer.unload();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("cache optimization", () => {
|
||||
test("should skip processing files with valid cache", async () => {
|
||||
const file = createMockFile("test.md", 1000);
|
||||
const tasks = [
|
||||
{
|
||||
id: "task1",
|
||||
content: "Test task",
|
||||
filePath: file.path,
|
||||
line: 1,
|
||||
completed: false,
|
||||
status: " ",
|
||||
originalMarkdown: "- [ ] Test task",
|
||||
metadata: {
|
||||
tags: [],
|
||||
project: undefined,
|
||||
context: undefined,
|
||||
priority: undefined,
|
||||
dueDate: undefined,
|
||||
startDate: undefined,
|
||||
scheduledDate: undefined,
|
||||
completedDate: undefined,
|
||||
cancelledDate: undefined,
|
||||
createdDate: undefined,
|
||||
recurrence: undefined,
|
||||
dependsOn: [],
|
||||
onCompletion: undefined,
|
||||
taskId: undefined,
|
||||
children: [],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// Pre-populate cache
|
||||
indexer.updateIndexWithTasks(file.path, tasks, file.stat.mtime);
|
||||
|
||||
// Process file - should use cache
|
||||
const result = await workerManager.processFile(file);
|
||||
|
||||
// Should return cached tasks without calling vault.cachedRead
|
||||
expect(result).toEqual(tasks);
|
||||
expect(mockVault.cachedRead).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("should process files when cache is invalid", async () => {
|
||||
const file = createMockFile("test.md", 2000);
|
||||
const oldTasks = [
|
||||
{
|
||||
id: "task1",
|
||||
content: "Old task",
|
||||
filePath: file.path,
|
||||
line: 1,
|
||||
completed: false,
|
||||
status: " ",
|
||||
originalMarkdown: "- [ ] Old task",
|
||||
metadata: {
|
||||
tags: [],
|
||||
project: undefined,
|
||||
context: undefined,
|
||||
priority: undefined,
|
||||
dueDate: undefined,
|
||||
startDate: undefined,
|
||||
scheduledDate: undefined,
|
||||
completedDate: undefined,
|
||||
cancelledDate: undefined,
|
||||
createdDate: undefined,
|
||||
recurrence: undefined,
|
||||
dependsOn: [],
|
||||
onCompletion: undefined,
|
||||
taskId: undefined,
|
||||
children: [],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// Pre-populate cache with older mtime
|
||||
indexer.updateIndexWithTasks(file.path, oldTasks, 1000);
|
||||
|
||||
// Mock worker processing (since we can't easily test actual worker)
|
||||
// This would normally go through the worker, but for testing we'll simulate
|
||||
// the file being processed
|
||||
expect(indexer.hasValidCache(file.path, file.stat.mtime)).toBe(false);
|
||||
});
|
||||
|
||||
test("should optimize batch processing", async () => {
|
||||
const files = [
|
||||
createMockFile("cached1.md", 1000),
|
||||
createMockFile("cached2.md", 1000),
|
||||
createMockFile("new.md", 2000),
|
||||
];
|
||||
|
||||
const cachedTasks = [
|
||||
{
|
||||
id: "cached-task",
|
||||
content: "Cached task",
|
||||
filePath: "cached1.md",
|
||||
line: 1,
|
||||
completed: false,
|
||||
status: " ",
|
||||
originalMarkdown: "- [ ] Cached task",
|
||||
metadata: {
|
||||
tags: [],
|
||||
project: undefined,
|
||||
context: undefined,
|
||||
priority: undefined,
|
||||
dueDate: undefined,
|
||||
startDate: undefined,
|
||||
scheduledDate: undefined,
|
||||
completedDate: undefined,
|
||||
cancelledDate: undefined,
|
||||
createdDate: undefined,
|
||||
recurrence: undefined,
|
||||
dependsOn: [],
|
||||
onCompletion: undefined,
|
||||
taskId: undefined,
|
||||
children: [],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// Pre-populate cache for first two files
|
||||
indexer.updateIndexWithTasks("cached1.md", cachedTasks, 1000);
|
||||
indexer.updateIndexWithTasks("cached2.md", cachedTasks, 1000);
|
||||
|
||||
// Process batch
|
||||
const result = await workerManager.processBatch(files);
|
||||
|
||||
// Should have results for cached files
|
||||
expect(result.has("cached1.md")).toBe(true);
|
||||
expect(result.has("cached2.md")).toBe(true);
|
||||
expect(result.get("cached1.md")).toEqual(cachedTasks);
|
||||
expect(result.get("cached2.md")).toEqual(cachedTasks);
|
||||
|
||||
// Check statistics
|
||||
const stats = workerManager.getStats();
|
||||
expect(stats.filesSkipped).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("configuration", () => {
|
||||
test("should respect mtime optimization setting", () => {
|
||||
// Create worker manager with optimization disabled
|
||||
const workerManagerDisabled = new TaskWorkerManager(mockVault, mockMetadataCache, {
|
||||
settings: {
|
||||
fileParsingConfig: {
|
||||
enableMtimeOptimization: false,
|
||||
mtimeCacheSize: 1000,
|
||||
enableFileMetadataParsing: false,
|
||||
metadataFieldsToParseAsTasks: [],
|
||||
enableTagBasedTaskParsing: false,
|
||||
tagsToParseAsTasks: [],
|
||||
taskContentFromMetadata: "title",
|
||||
defaultTaskStatus: " ",
|
||||
enableWorkerProcessing: true,
|
||||
},
|
||||
preferMetadataFormat: "tasks",
|
||||
useDailyNotePathAsDate: false,
|
||||
dailyNoteFormat: "yyyy-MM-dd",
|
||||
useAsDateType: "due",
|
||||
dailyNotePath: "",
|
||||
ignoreHeading: "",
|
||||
focusHeading: "",
|
||||
fileMetadataInheritance: undefined,
|
||||
},
|
||||
});
|
||||
|
||||
workerManagerDisabled.setTaskIndexer(indexer);
|
||||
|
||||
const file = createMockFile("test.md", 1000);
|
||||
|
||||
// Pre-populate cache
|
||||
indexer.updateIndexWithTasks(file.path, [], 1000);
|
||||
|
||||
// Should always process when optimization is disabled
|
||||
// (We can't easily test the private shouldProcessFile method,
|
||||
// but this demonstrates the configuration is respected)
|
||||
expect(indexer.hasValidCache(file.path, file.stat.mtime)).toBe(false); // No tasks in cache
|
||||
|
||||
workerManagerDisabled.unload();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -494,6 +494,10 @@ export interface FileParsingConfiguration {
|
|||
defaultTaskStatus: string;
|
||||
/** Whether to use worker for file parsing performance */
|
||||
enableWorkerProcessing: boolean;
|
||||
/** Whether to enable mtime-based cache optimization */
|
||||
enableMtimeOptimization: boolean;
|
||||
/** Maximum number of files to track in mtime cache */
|
||||
mtimeCacheSize: number;
|
||||
}
|
||||
|
||||
/** Timeline Sidebar Settings */
|
||||
|
|
@ -926,6 +930,8 @@ export const DEFAULT_SETTINGS: TaskProgressBarSettings = {
|
|||
taskContentFromMetadata: "title",
|
||||
defaultTaskStatus: " ",
|
||||
enableWorkerProcessing: true,
|
||||
enableMtimeOptimization: true,
|
||||
mtimeCacheSize: 10000,
|
||||
},
|
||||
|
||||
// Date Settings
|
||||
|
|
|
|||
7
src/types/task.d.ts
vendored
7
src/types/task.d.ts
vendored
|
|
@ -109,7 +109,6 @@ export interface CanvasTaskMetadata extends StandardTaskMetadata {
|
|||
|
||||
/** Source type to distinguish canvas tasks */
|
||||
sourceType?: "canvas" | "markdown";
|
||||
|
||||
}
|
||||
|
||||
/** Task Genius Project interface */
|
||||
|
|
@ -213,6 +212,12 @@ export interface TaskCache {
|
|||
|
||||
/** Priority index: priority -> Set<taskIds> */
|
||||
priority: Map<number, Set<string>>;
|
||||
|
||||
/** File modification times: filePath -> mtime */
|
||||
fileMtimes: Map<string, number>;
|
||||
|
||||
/** File processed times: filePath -> processedTime */
|
||||
fileProcessedTimes: Map<string, number>;
|
||||
}
|
||||
|
||||
/** Task filter interface for querying tasks */
|
||||
|
|
|
|||
|
|
@ -160,6 +160,8 @@ export class TaskManager extends Component {
|
|||
settings: this.plugin.settings,
|
||||
}
|
||||
);
|
||||
// Set task indexer reference for cache checking
|
||||
this.workerManager.setTaskIndexer(this.indexer);
|
||||
this.log("Worker manager initialized");
|
||||
} catch (error) {
|
||||
console.error("Failed to initialize worker manager:", error);
|
||||
|
|
@ -663,6 +665,7 @@ export class TaskManager extends Component {
|
|||
this.indexer.updateIndexWithTasks(
|
||||
filePath,
|
||||
cacheItem.data
|
||||
// Note: mtime not available here, will be set when file is processed
|
||||
);
|
||||
this.log(
|
||||
`Preloaded ${cacheItem.data.length} tasks from cache for ${filePath}`
|
||||
|
|
@ -855,7 +858,8 @@ export class TaskManager extends Component {
|
|||
// Update index with cached data
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
cached.data
|
||||
cached.data,
|
||||
file.stat.mtime
|
||||
);
|
||||
this.log(
|
||||
`Loaded ${cached.data.length} tasks from cache for ${file.path}`
|
||||
|
|
@ -988,7 +992,11 @@ export class TaskManager extends Component {
|
|||
const tasks = await this.workerManager.processFile(file);
|
||||
|
||||
// Update the index with the tasks
|
||||
this.indexer.updateIndexWithTasks(file.path, tasks);
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
tasks,
|
||||
file.stat.mtime
|
||||
);
|
||||
|
||||
// Store tasks in cache if there are any
|
||||
if (tasks.length > 0) {
|
||||
|
|
@ -1039,7 +1047,11 @@ export class TaskManager extends Component {
|
|||
|
||||
console.log("tasks", tasks, file.path);
|
||||
// Update the index with the tasks
|
||||
this.indexer.updateIndexWithTasks(file.path, tasks);
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
tasks,
|
||||
file.stat.mtime
|
||||
);
|
||||
|
||||
// Store tasks in cache if there are any
|
||||
if (tasks.length > 0) {
|
||||
|
|
@ -1125,7 +1137,8 @@ export class TaskManager extends Component {
|
|||
// Update index with cached data
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
cached.data
|
||||
cached.data,
|
||||
file.stat.mtime
|
||||
);
|
||||
this.log(
|
||||
`Loaded ${cached.data.length} tasks from cache for ${file.path}`
|
||||
|
|
@ -1157,7 +1170,11 @@ export class TaskManager extends Component {
|
|||
);
|
||||
|
||||
// Update index with parsed tasks
|
||||
this.indexer.updateIndexWithTasks(file.path, tasks);
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
tasks,
|
||||
file.stat.mtime
|
||||
);
|
||||
|
||||
// Store to cache
|
||||
if (tasks.length > 0) {
|
||||
|
|
@ -1189,7 +1206,11 @@ export class TaskManager extends Component {
|
|||
file.path,
|
||||
content
|
||||
);
|
||||
this.indexer.updateIndexWithTasks(file.path, tasks);
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
tasks,
|
||||
file.stat.mtime
|
||||
);
|
||||
|
||||
if (tasks.length > 0) {
|
||||
await this.persister.storeFile(file.path, tasks);
|
||||
|
|
@ -1295,7 +1316,11 @@ export class TaskManager extends Component {
|
|||
);
|
||||
|
||||
// Update index with parsed tasks
|
||||
this.indexer.updateIndexWithTasks(file.path, tasks);
|
||||
this.indexer.updateIndexWithTasks(
|
||||
file.path,
|
||||
tasks,
|
||||
file.stat.mtime
|
||||
);
|
||||
|
||||
// Cache the results
|
||||
if (tasks.length > 0) {
|
||||
|
|
@ -1350,7 +1375,7 @@ export class TaskManager extends Component {
|
|||
* Remove a file from the index based on the old path
|
||||
*/
|
||||
private removeFileFromIndexByOldPath(oldPath: string): void {
|
||||
this.indexer.updateIndexWithTasks(oldPath, []);
|
||||
this.indexer.cleanupFileCache(oldPath);
|
||||
try {
|
||||
this.persister.removeFile(oldPath);
|
||||
this.log(`Removed ${oldPath} from cache`);
|
||||
|
|
@ -1370,7 +1395,7 @@ export class TaskManager extends Component {
|
|||
*/
|
||||
private removeFileFromIndex(file: TFile): void {
|
||||
// 使用 indexer 的方法来删除文件
|
||||
this.indexer.updateIndexWithTasks(file.path, []);
|
||||
this.indexer.cleanupFileCache(file.path);
|
||||
|
||||
// 从缓存中删除文件
|
||||
try {
|
||||
|
|
@ -2899,16 +2924,22 @@ export class TaskManager extends Component {
|
|||
const {
|
||||
clearProjectCaches = true,
|
||||
preserveValidCaches = false,
|
||||
logCacheStats = false
|
||||
logCacheStats = false,
|
||||
} = options || {};
|
||||
|
||||
this.log(`Force reindexing all tasks (clearProjectCaches: ${clearProjectCaches}, preserveValidCaches: ${preserveValidCaches})`);
|
||||
this.log(
|
||||
`Force reindexing all tasks (clearProjectCaches: ${clearProjectCaches}, preserveValidCaches: ${preserveValidCaches})`
|
||||
);
|
||||
|
||||
// Log cache statistics before clearing if requested
|
||||
if (logCacheStats && this.taskParsingService) {
|
||||
try {
|
||||
const beforeStats = this.taskParsingService.getDetailedCacheStats();
|
||||
this.log("Cache statistics before clearing: " + JSON.stringify(beforeStats.summary, null, 2));
|
||||
const beforeStats =
|
||||
this.taskParsingService.getDetailedCacheStats();
|
||||
this.log(
|
||||
"Cache statistics before clearing: " +
|
||||
JSON.stringify(beforeStats.summary, null, 2)
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn("Failed to get cache statistics:", error);
|
||||
}
|
||||
|
|
@ -2926,13 +2957,19 @@ export class TaskManager extends Component {
|
|||
if (preserveValidCaches) {
|
||||
// Smart cache clearing - only clear stale entries
|
||||
if ((this.taskParsingService as any).projectConfigManager) {
|
||||
const clearedCount = await (this.taskParsingService as any).projectConfigManager.clearStaleEntries();
|
||||
this.log(`Smart cache clearing: removed ${clearedCount} stale entries`);
|
||||
const clearedCount = await (
|
||||
this.taskParsingService as any
|
||||
).projectConfigManager.clearStaleEntries();
|
||||
this.log(
|
||||
`Smart cache clearing: removed ${clearedCount} stale entries`
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Full cache clearing (default behavior)
|
||||
this.taskParsingService.clearAllCaches();
|
||||
this.log("Cleared all project-related caches (config, data, metadata)");
|
||||
this.log(
|
||||
"Cleared all project-related caches (config, data, metadata)"
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error clearing project caches:", error);
|
||||
|
|
@ -2987,8 +3024,12 @@ export class TaskManager extends Component {
|
|||
// Log cache statistics after rebuilding if requested
|
||||
if (logCacheStats && this.taskParsingService) {
|
||||
try {
|
||||
const afterStats = this.taskParsingService.getDetailedCacheStats();
|
||||
this.log("Cache statistics after rebuilding: " + JSON.stringify(afterStats.summary, null, 2));
|
||||
const afterStats =
|
||||
this.taskParsingService.getDetailedCacheStats();
|
||||
this.log(
|
||||
"Cache statistics after rebuilding: " +
|
||||
JSON.stringify(afterStats.summary, null, 2)
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn("Failed to get final cache statistics:", error);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,6 +98,8 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
|
|||
onCompletion: new Map<string, Set<string>>(),
|
||||
dependsOn: new Map<string, Set<string>>(),
|
||||
taskId: new Map<string, Set<string>>(),
|
||||
fileMtimes: new Map<string, number>(),
|
||||
fileProcessedTimes: new Map<string, number>(),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -238,7 +240,11 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
|
|||
* Update the index with tasks parsed by external components
|
||||
* This is the primary method for updating the index
|
||||
*/
|
||||
public updateIndexWithTasks(filePath: string, tasks: Task[]): void {
|
||||
public updateIndexWithTasks(
|
||||
filePath: string,
|
||||
tasks: Task[],
|
||||
fileMtime?: number
|
||||
): void {
|
||||
// Remove existing tasks for this file first
|
||||
this.removeFileFromIndex(filePath);
|
||||
|
||||
|
|
@ -257,6 +263,11 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
|
|||
// Update file index
|
||||
this.taskCache.files.set(filePath, fileTaskIds);
|
||||
this.lastIndexTime.set(filePath, Date.now());
|
||||
|
||||
// Update file mtime if provided
|
||||
if (fileMtime !== undefined) {
|
||||
this.updateFileMtime(filePath, fileMtime);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1052,6 +1063,75 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
|
|||
this.taskCache = this.initEmptyCache();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file has changed since last processing
|
||||
*/
|
||||
public isFileChanged(filePath: string, currentMtime: number): boolean {
|
||||
const lastMtime = this.taskCache.fileMtimes.get(filePath);
|
||||
return lastMtime === undefined || lastMtime < currentMtime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last known modification time for a file
|
||||
*/
|
||||
public getFileLastMtime(filePath: string): number | undefined {
|
||||
return this.taskCache.fileMtimes.get(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the modification time for a file
|
||||
*/
|
||||
public updateFileMtime(filePath: string, mtime: number): void {
|
||||
this.taskCache.fileMtimes.set(filePath, mtime);
|
||||
this.taskCache.fileProcessedTimes.set(filePath, Date.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we have valid cache for a file
|
||||
*/
|
||||
public hasValidCache(filePath: string, currentMtime: number): boolean {
|
||||
// Check if file has tasks in cache
|
||||
const hasTasksInCache = this.taskCache.files.has(filePath);
|
||||
|
||||
// Check if file hasn't changed
|
||||
const hasNotChanged = !this.isFileChanged(filePath, currentMtime);
|
||||
|
||||
return hasTasksInCache && hasNotChanged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up cache for a specific file
|
||||
*/
|
||||
public cleanupFileCache(filePath: string): void {
|
||||
// Remove from file mtime cache
|
||||
this.taskCache.fileMtimes.delete(filePath);
|
||||
this.taskCache.fileProcessedTimes.delete(filePath);
|
||||
|
||||
// Remove from other caches (handled by existing removeFileFromIndex)
|
||||
this.removeFileFromIndex(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate cache consistency and fix any issues
|
||||
*/
|
||||
public validateCacheConsistency(): void {
|
||||
// Check for files in mtime cache but not in file index
|
||||
for (const filePath of this.taskCache.fileMtimes.keys()) {
|
||||
if (!this.taskCache.files.has(filePath)) {
|
||||
this.taskCache.fileMtimes.delete(filePath);
|
||||
this.taskCache.fileProcessedTimes.delete(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for files in file index but not in mtime cache
|
||||
for (const filePath of this.taskCache.files.keys()) {
|
||||
if (!this.taskCache.fileMtimes.has(filePath)) {
|
||||
// This is acceptable - mtime might not be set for older cache entries
|
||||
// We don't need to remove the file from index
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the cache from an external source (e.g. persisted cache)
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@ import {
|
|||
TaskParseResult,
|
||||
} from "./TaskIndexWorkerMessage";
|
||||
import { FileMetadataTaskParser } from "./FileMetadataTaskParser";
|
||||
import { FileParsingConfiguration, FileMetadataInheritanceConfig } from "../../common/setting-definition";
|
||||
import {
|
||||
FileParsingConfiguration,
|
||||
FileMetadataInheritanceConfig,
|
||||
} from "../../common/setting-definition";
|
||||
|
||||
// Import worker and utilities
|
||||
// @ts-ignore Ignore type error for worker import
|
||||
|
|
@ -109,6 +112,8 @@ interface TaskMetadata {
|
|||
mtime: number;
|
||||
size: number;
|
||||
};
|
||||
/** Whether this metadata came from cache */
|
||||
fromCache?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -155,6 +160,14 @@ export class TaskWorkerManager extends Component {
|
|||
private fileMetadataParser?: FileMetadataTaskParser;
|
||||
/** Whether workers have been initialized to prevent multiple initialization */
|
||||
private initialized: boolean = false;
|
||||
/** Reference to task indexer for cache checking */
|
||||
private taskIndexer?: any;
|
||||
/** Performance statistics */
|
||||
private stats = {
|
||||
filesSkipped: 0,
|
||||
filesProcessed: 0,
|
||||
cacheHitRatio: 0,
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a new worker pool
|
||||
|
|
@ -304,6 +317,73 @@ export class TaskWorkerManager extends Component {
|
|||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the task indexer reference for cache checking
|
||||
*/
|
||||
public setTaskIndexer(taskIndexer: any): void {
|
||||
this.taskIndexer = taskIndexer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update cache hit ratio statistics
|
||||
*/
|
||||
private updateCacheHitRatio(): void {
|
||||
const totalFiles = this.stats.filesSkipped + this.stats.filesProcessed;
|
||||
this.stats.cacheHitRatio =
|
||||
totalFiles > 0 ? this.stats.filesSkipped / totalFiles : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get performance statistics
|
||||
*/
|
||||
public getStats() {
|
||||
return { ...this.stats };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file should be processed (not in valid cache)
|
||||
*/
|
||||
private shouldProcessFile(file: TFile): boolean {
|
||||
if (!this.taskIndexer) {
|
||||
return true; // No indexer, always process
|
||||
}
|
||||
|
||||
// Check if mtime optimization is enabled
|
||||
if (
|
||||
!this.options.settings?.fileParsingConfig?.enableMtimeOptimization
|
||||
) {
|
||||
return true; // Optimization disabled, always process
|
||||
}
|
||||
|
||||
return !this.taskIndexer.hasValidCache(file.path, file.stat.mtime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached tasks for a file if available
|
||||
*/
|
||||
private getCachedTasksForFile(filePath: string): Task[] | null {
|
||||
if (!this.taskIndexer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const taskIds = this.taskIndexer.getCache().files.get(filePath);
|
||||
if (!taskIds) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const tasks: Task[] = [];
|
||||
const taskCache = this.taskIndexer.getCache().tasks;
|
||||
|
||||
for (const taskId of taskIds) {
|
||||
const task = taskCache.get(taskId);
|
||||
if (task) {
|
||||
tasks.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
return tasks.length > 0 ? tasks : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single file for tasks
|
||||
*/
|
||||
|
|
@ -315,6 +395,19 @@ export class TaskWorkerManager extends Component {
|
|||
let existing = this.outstanding.get(file.path);
|
||||
if (existing) return existing;
|
||||
|
||||
// Check if we can use cached results
|
||||
if (!this.shouldProcessFile(file)) {
|
||||
const cachedTasks = this.getCachedTasksForFile(file.path);
|
||||
if (cachedTasks) {
|
||||
this.stats.filesSkipped++;
|
||||
this.updateCacheHitRatio();
|
||||
this.log(
|
||||
`Using cached tasks for ${file.path} (${cachedTasks.length} tasks)`
|
||||
);
|
||||
return Promise.resolve(cachedTasks);
|
||||
}
|
||||
}
|
||||
|
||||
let promise = deferred<Task[]>();
|
||||
this.outstanding.set(file.path, promise);
|
||||
|
||||
|
|
@ -354,14 +447,44 @@ export class TaskWorkerManager extends Component {
|
|||
return new Map<string, Task[]>();
|
||||
}
|
||||
|
||||
// Pre-filter files: separate cached from uncached
|
||||
const filesToProcess: TFile[] = [];
|
||||
const resultMap = new Map<string, Task[]>();
|
||||
let cachedCount = 0;
|
||||
|
||||
for (const file of files) {
|
||||
if (!this.shouldProcessFile(file)) {
|
||||
const cachedTasks = this.getCachedTasksForFile(file.path);
|
||||
if (cachedTasks) {
|
||||
resultMap.set(file.path, cachedTasks);
|
||||
cachedCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
filesToProcess.push(file);
|
||||
}
|
||||
|
||||
this.log(
|
||||
`Batch processing: ${cachedCount} files from cache, ${
|
||||
filesToProcess.length
|
||||
} files to process (cache hit ratio: ${
|
||||
cachedCount > 0
|
||||
? ((cachedCount / files.length) * 100).toFixed(1)
|
||||
: 0
|
||||
}%)`
|
||||
);
|
||||
|
||||
if (filesToProcess.length === 0) {
|
||||
return resultMap; // All files were cached
|
||||
}
|
||||
|
||||
this.isProcessingBatch = true;
|
||||
this.processedFiles = 0;
|
||||
this.totalFilesToProcess = files.length;
|
||||
this.totalFilesToProcess = filesToProcess.length;
|
||||
|
||||
this.log(`Processing batch of ${files.length} files`);
|
||||
|
||||
// 创建一个结果映射
|
||||
const resultMap = new Map<string, Task[]>();
|
||||
this.log(
|
||||
`Processing batch of ${filesToProcess.length} files (${cachedCount} cached)`
|
||||
);
|
||||
|
||||
try {
|
||||
// 将文件分成更小的批次,避免一次性提交太多任务
|
||||
|
|
@ -397,8 +520,8 @@ export class TaskWorkerManager extends Component {
|
|||
}
|
||||
};
|
||||
|
||||
for (let i = 0; i < files.length; i += batchSize) {
|
||||
const subBatch = files.slice(i, i + batchSize);
|
||||
for (let i = 0; i < filesToProcess.length; i += batchSize) {
|
||||
const subBatch = filesToProcess.slice(i, i + batchSize);
|
||||
|
||||
// 为子批次创建处理任务并添加到队列
|
||||
processingQueue.push(async () => {
|
||||
|
|
@ -450,7 +573,9 @@ export class TaskWorkerManager extends Component {
|
|||
console.error("Error during batch processing:", error);
|
||||
} finally {
|
||||
this.isProcessingBatch = false;
|
||||
this.log(`Completed batch processing of ${files.length} files`);
|
||||
this.log(
|
||||
`Completed batch processing of ${files.length} files (${cachedCount} from cache, ${filesToProcess.length} processed)`
|
||||
);
|
||||
}
|
||||
|
||||
return resultMap;
|
||||
|
|
@ -676,6 +801,10 @@ export class TaskWorkerManager extends Component {
|
|||
|
||||
promise.resolve(allTasks);
|
||||
this.outstanding.delete(file.path);
|
||||
|
||||
// Update statistics
|
||||
this.stats.filesProcessed++;
|
||||
this.updateCacheHitRatio();
|
||||
} else if (data.type === "batchResult") {
|
||||
// For batch results, we handle differently as we don't have tasks directly
|
||||
promise.reject(
|
||||
|
|
|
|||
Loading…
Reference in a new issue