fix: better performance

This commit is contained in:
quorafind 2025-07-16 15:04:51 +08:00
parent b0868745b3
commit 024fdc7dff
8 changed files with 725 additions and 29 deletions

View file

@ -25,6 +25,8 @@ describe("FileMetadataTaskParser", () => {
taskContentFromMetadata: "title",
defaultTaskStatus: " ",
enableWorkerProcessing: true,
enableMtimeOptimization: false,
mtimeCacheSize: 1000,
};
parser = new FileMetadataTaskParser(config);
});
@ -206,6 +208,8 @@ describe("FileMetadataTaskParser", () => {
taskContentFromMetadata: "title",
defaultTaskStatus: " ",
enableWorkerProcessing: true,
enableMtimeOptimization: false,
mtimeCacheSize: 1000,
};
const disabledParser = new FileMetadataTaskParser(disabledConfig);
@ -358,6 +362,8 @@ describe("FileMetadataTaskUpdater", () => {
taskContentFromMetadata: "title",
defaultTaskStatus: " ",
enableWorkerProcessing: true,
enableMtimeOptimization: false,
mtimeCacheSize: 1000,
};
// Mock Obsidian App

View file

@ -0,0 +1,172 @@
/**
* Tests for TaskIndexer mtime-based caching functionality
*/
import { TaskIndexer } from "../utils/import/TaskIndexer";
import { Task } from "../types/task";
// Mock obsidian Component class
jest.mock("obsidian", () => ({
...jest.requireActual("obsidian"),
Component: class {
registerEvent = jest.fn();
unload = jest.fn();
},
TFile: jest.fn(),
}));
// Mock dependencies
const mockApp = {} as any;
const mockVault = {
on: jest.fn().mockReturnValue({}),
off: jest.fn(),
} as any;
const mockMetadataCache = {} as any;
describe("TaskIndexer mtime functionality", () => {
let indexer: TaskIndexer;
beforeEach(() => {
indexer = new TaskIndexer(mockApp, mockVault, mockMetadataCache);
});
afterEach(() => {
if (indexer && typeof indexer.unload === 'function') {
indexer.unload();
}
});
describe("mtime comparison", () => {
test("should detect file changes when mtime is newer", () => {
const filePath = "test.md";
const oldMtime = 1000;
const newMtime = 2000;
// Set initial mtime
indexer.updateFileMtime(filePath, oldMtime);
// Check if file is changed with newer mtime
expect(indexer.isFileChanged(filePath, newMtime)).toBe(true);
});
test("should not detect changes when mtime is same", () => {
const filePath = "test.md";
const mtime = 1000;
// Set initial mtime
indexer.updateFileMtime(filePath, mtime);
// Check if file is changed with same mtime
expect(indexer.isFileChanged(filePath, mtime)).toBe(false);
});
test("should detect changes for unknown files", () => {
const filePath = "unknown.md";
const mtime = 1000;
// Check if unknown file is considered changed
expect(indexer.isFileChanged(filePath, mtime)).toBe(true);
});
});
describe("cache validation", () => {
test("should have valid cache when file hasn't changed and has tasks", () => {
const filePath = "test.md";
const mtime = 1000;
const tasks: Task[] = [
{
id: "task1",
content: "Test task",
filePath,
line: 1,
completed: false,
status: " ",
originalMarkdown: "- [ ] Test task",
metadata: {
tags: [],
project: undefined,
context: undefined,
priority: undefined,
dueDate: undefined,
startDate: undefined,
scheduledDate: undefined,
completedDate: undefined,
cancelledDate: undefined,
createdDate: undefined,
recurrence: undefined,
dependsOn: [],
onCompletion: undefined,
taskId: undefined,
children: [],
},
},
];
// Add tasks and set mtime
indexer.updateIndexWithTasks(filePath, tasks, mtime);
// Check if cache is valid
expect(indexer.hasValidCache(filePath, mtime)).toBe(true);
});
test("should not have valid cache when file has changed", () => {
const filePath = "test.md";
const oldMtime = 1000;
const newMtime = 2000;
const tasks: Task[] = [];
// Add tasks with old mtime
indexer.updateIndexWithTasks(filePath, tasks, oldMtime);
// Check if cache is invalid with new mtime
expect(indexer.hasValidCache(filePath, newMtime)).toBe(false);
});
test("should not have valid cache when no tasks exist", () => {
const filePath = "test.md";
const mtime = 1000;
// Don't add any tasks, just set mtime
indexer.updateFileMtime(filePath, mtime);
// Check if cache is invalid when no tasks exist
expect(indexer.hasValidCache(filePath, mtime)).toBe(false);
});
});
describe("cache cleanup", () => {
test("should clean up file cache properly", () => {
const filePath = "test.md";
const mtime = 1000;
const tasks: Task[] = [];
// Add tasks and set mtime
indexer.updateIndexWithTasks(filePath, tasks, mtime);
// Verify cache exists
expect(indexer.getFileLastMtime(filePath)).toBe(mtime);
// Clean up cache
indexer.cleanupFileCache(filePath);
// Verify cache is cleaned
expect(indexer.getFileLastMtime(filePath)).toBeUndefined();
});
});
describe("cache consistency", () => {
test("should validate and fix cache consistency", () => {
const filePath = "test.md";
const mtime = 1000;
// Manually add mtime without tasks (inconsistent state)
indexer.updateFileMtime(filePath, mtime);
// Validate consistency (should clean up orphaned mtime)
indexer.validateCacheConsistency();
// Verify orphaned mtime is cleaned up
expect(indexer.getFileLastMtime(filePath)).toBeUndefined();
});
});
});

View file

@ -0,0 +1,257 @@
/**
* Integration tests for TaskWorkerManager mtime optimization
*/
import { TaskWorkerManager } from "../utils/workers/TaskWorkerManager";
import { TaskIndexer } from "../utils/import/TaskIndexer";
import { TFile } from "obsidian";
// Mock dependencies
const mockVault = {
cachedRead: jest.fn(),
} as any;
const mockMetadataCache = {
getFileCache: jest.fn(),
} as any;
const mockApp = {} as any;
// Mock TFile
const createMockFile = (path: string, mtime: number): TFile => ({
path,
stat: { mtime, ctime: mtime, size: 100 },
extension: "md",
name: path.split("/").pop() || path,
basename: path.split("/").pop()?.replace(".md", "") || path,
} as any);
describe("TaskWorkerManager mtime optimization", () => {
let workerManager: TaskWorkerManager;
let indexer: TaskIndexer;
beforeEach(() => {
// Create indexer
indexer = new TaskIndexer(mockApp, mockVault, mockMetadataCache);
// Create worker manager with mtime optimization enabled
workerManager = new TaskWorkerManager(mockVault, mockMetadataCache, {
maxWorkers: 1,
settings: {
fileParsingConfig: {
enableMtimeOptimization: true,
mtimeCacheSize: 1000,
enableFileMetadataParsing: false,
metadataFieldsToParseAsTasks: [],
enableTagBasedTaskParsing: false,
tagsToParseAsTasks: [],
taskContentFromMetadata: "title",
defaultTaskStatus: " ",
enableWorkerProcessing: true,
},
preferMetadataFormat: "tasks",
useDailyNotePathAsDate: false,
dailyNoteFormat: "yyyy-MM-dd",
useAsDateType: "due",
dailyNotePath: "",
ignoreHeading: "",
focusHeading: "",
fileMetadataInheritance: undefined,
},
});
// Set indexer reference
workerManager.setTaskIndexer(indexer);
// Mock vault.cachedRead to return empty content
mockVault.cachedRead.mockResolvedValue("");
mockMetadataCache.getFileCache.mockReturnValue(null);
});
afterEach(() => {
workerManager.unload();
indexer.unload();
jest.clearAllMocks();
});
describe("cache optimization", () => {
test("should skip processing files with valid cache", async () => {
const file = createMockFile("test.md", 1000);
const tasks = [
{
id: "task1",
content: "Test task",
filePath: file.path,
line: 1,
completed: false,
status: " ",
originalMarkdown: "- [ ] Test task",
metadata: {
tags: [],
project: undefined,
context: undefined,
priority: undefined,
dueDate: undefined,
startDate: undefined,
scheduledDate: undefined,
completedDate: undefined,
cancelledDate: undefined,
createdDate: undefined,
recurrence: undefined,
dependsOn: [],
onCompletion: undefined,
taskId: undefined,
children: [],
},
},
];
// Pre-populate cache
indexer.updateIndexWithTasks(file.path, tasks, file.stat.mtime);
// Process file - should use cache
const result = await workerManager.processFile(file);
// Should return cached tasks without calling vault.cachedRead
expect(result).toEqual(tasks);
expect(mockVault.cachedRead).not.toHaveBeenCalled();
});
test("should process files when cache is invalid", async () => {
const file = createMockFile("test.md", 2000);
const oldTasks = [
{
id: "task1",
content: "Old task",
filePath: file.path,
line: 1,
completed: false,
status: " ",
originalMarkdown: "- [ ] Old task",
metadata: {
tags: [],
project: undefined,
context: undefined,
priority: undefined,
dueDate: undefined,
startDate: undefined,
scheduledDate: undefined,
completedDate: undefined,
cancelledDate: undefined,
createdDate: undefined,
recurrence: undefined,
dependsOn: [],
onCompletion: undefined,
taskId: undefined,
children: [],
},
},
];
// Pre-populate cache with older mtime
indexer.updateIndexWithTasks(file.path, oldTasks, 1000);
// Mock worker processing (since we can't easily test actual worker)
// This would normally go through the worker, but for testing we'll simulate
// the file being processed
expect(indexer.hasValidCache(file.path, file.stat.mtime)).toBe(false);
});
test("should optimize batch processing", async () => {
const files = [
createMockFile("cached1.md", 1000),
createMockFile("cached2.md", 1000),
createMockFile("new.md", 2000),
];
const cachedTasks = [
{
id: "cached-task",
content: "Cached task",
filePath: "cached1.md",
line: 1,
completed: false,
status: " ",
originalMarkdown: "- [ ] Cached task",
metadata: {
tags: [],
project: undefined,
context: undefined,
priority: undefined,
dueDate: undefined,
startDate: undefined,
scheduledDate: undefined,
completedDate: undefined,
cancelledDate: undefined,
createdDate: undefined,
recurrence: undefined,
dependsOn: [],
onCompletion: undefined,
taskId: undefined,
children: [],
},
},
];
// Pre-populate cache for first two files
indexer.updateIndexWithTasks("cached1.md", cachedTasks, 1000);
indexer.updateIndexWithTasks("cached2.md", cachedTasks, 1000);
// Process batch
const result = await workerManager.processBatch(files);
// Should have results for cached files
expect(result.has("cached1.md")).toBe(true);
expect(result.has("cached2.md")).toBe(true);
expect(result.get("cached1.md")).toEqual(cachedTasks);
expect(result.get("cached2.md")).toEqual(cachedTasks);
// Check statistics
const stats = workerManager.getStats();
expect(stats.filesSkipped).toBe(2);
});
});
describe("configuration", () => {
test("should respect mtime optimization setting", () => {
// Create worker manager with optimization disabled
const workerManagerDisabled = new TaskWorkerManager(mockVault, mockMetadataCache, {
settings: {
fileParsingConfig: {
enableMtimeOptimization: false,
mtimeCacheSize: 1000,
enableFileMetadataParsing: false,
metadataFieldsToParseAsTasks: [],
enableTagBasedTaskParsing: false,
tagsToParseAsTasks: [],
taskContentFromMetadata: "title",
defaultTaskStatus: " ",
enableWorkerProcessing: true,
},
preferMetadataFormat: "tasks",
useDailyNotePathAsDate: false,
dailyNoteFormat: "yyyy-MM-dd",
useAsDateType: "due",
dailyNotePath: "",
ignoreHeading: "",
focusHeading: "",
fileMetadataInheritance: undefined,
},
});
workerManagerDisabled.setTaskIndexer(indexer);
const file = createMockFile("test.md", 1000);
// Pre-populate cache
indexer.updateIndexWithTasks(file.path, [], 1000);
// Should always process when optimization is disabled
// (We can't easily test the private shouldProcessFile method,
// but this demonstrates the configuration is respected)
expect(indexer.hasValidCache(file.path, file.stat.mtime)).toBe(false); // No tasks in cache
workerManagerDisabled.unload();
});
});
});

View file

@ -494,6 +494,10 @@ export interface FileParsingConfiguration {
defaultTaskStatus: string;
/** Whether to use worker for file parsing performance */
enableWorkerProcessing: boolean;
/** Whether to enable mtime-based cache optimization */
enableMtimeOptimization: boolean;
/** Maximum number of files to track in mtime cache */
mtimeCacheSize: number;
}
/** Timeline Sidebar Settings */
@ -926,6 +930,8 @@ export const DEFAULT_SETTINGS: TaskProgressBarSettings = {
taskContentFromMetadata: "title",
defaultTaskStatus: " ",
enableWorkerProcessing: true,
enableMtimeOptimization: true,
mtimeCacheSize: 10000,
},
// Date Settings

7
src/types/task.d.ts vendored
View file

@ -109,7 +109,6 @@ export interface CanvasTaskMetadata extends StandardTaskMetadata {
/** Source type to distinguish canvas tasks */
sourceType?: "canvas" | "markdown";
}
/** Task Genius Project interface */
@ -213,6 +212,12 @@ export interface TaskCache {
/** Priority index: priority -> Set<taskIds> */
priority: Map<number, Set<string>>;
/** File modification times: filePath -> mtime */
fileMtimes: Map<string, number>;
/** File processed times: filePath -> processedTime */
fileProcessedTimes: Map<string, number>;
}
/** Task filter interface for querying tasks */

View file

@ -160,6 +160,8 @@ export class TaskManager extends Component {
settings: this.plugin.settings,
}
);
// Set task indexer reference for cache checking
this.workerManager.setTaskIndexer(this.indexer);
this.log("Worker manager initialized");
} catch (error) {
console.error("Failed to initialize worker manager:", error);
@ -663,6 +665,7 @@ export class TaskManager extends Component {
this.indexer.updateIndexWithTasks(
filePath,
cacheItem.data
// Note: mtime not available here, will be set when file is processed
);
this.log(
`Preloaded ${cacheItem.data.length} tasks from cache for ${filePath}`
@ -855,7 +858,8 @@ export class TaskManager extends Component {
// Update index with cached data
this.indexer.updateIndexWithTasks(
file.path,
cached.data
cached.data,
file.stat.mtime
);
this.log(
`Loaded ${cached.data.length} tasks from cache for ${file.path}`
@ -988,7 +992,11 @@ export class TaskManager extends Component {
const tasks = await this.workerManager.processFile(file);
// Update the index with the tasks
this.indexer.updateIndexWithTasks(file.path, tasks);
this.indexer.updateIndexWithTasks(
file.path,
tasks,
file.stat.mtime
);
// Store tasks in cache if there are any
if (tasks.length > 0) {
@ -1039,7 +1047,11 @@ export class TaskManager extends Component {
console.log("tasks", tasks, file.path);
// Update the index with the tasks
this.indexer.updateIndexWithTasks(file.path, tasks);
this.indexer.updateIndexWithTasks(
file.path,
tasks,
file.stat.mtime
);
// Store tasks in cache if there are any
if (tasks.length > 0) {
@ -1125,7 +1137,8 @@ export class TaskManager extends Component {
// Update index with cached data
this.indexer.updateIndexWithTasks(
file.path,
cached.data
cached.data,
file.stat.mtime
);
this.log(
`Loaded ${cached.data.length} tasks from cache for ${file.path}`
@ -1157,7 +1170,11 @@ export class TaskManager extends Component {
);
// Update index with parsed tasks
this.indexer.updateIndexWithTasks(file.path, tasks);
this.indexer.updateIndexWithTasks(
file.path,
tasks,
file.stat.mtime
);
// Store to cache
if (tasks.length > 0) {
@ -1189,7 +1206,11 @@ export class TaskManager extends Component {
file.path,
content
);
this.indexer.updateIndexWithTasks(file.path, tasks);
this.indexer.updateIndexWithTasks(
file.path,
tasks,
file.stat.mtime
);
if (tasks.length > 0) {
await this.persister.storeFile(file.path, tasks);
@ -1295,7 +1316,11 @@ export class TaskManager extends Component {
);
// Update index with parsed tasks
this.indexer.updateIndexWithTasks(file.path, tasks);
this.indexer.updateIndexWithTasks(
file.path,
tasks,
file.stat.mtime
);
// Cache the results
if (tasks.length > 0) {
@ -1350,7 +1375,7 @@ export class TaskManager extends Component {
* Remove a file from the index based on the old path
*/
private removeFileFromIndexByOldPath(oldPath: string): void {
this.indexer.updateIndexWithTasks(oldPath, []);
this.indexer.cleanupFileCache(oldPath);
try {
this.persister.removeFile(oldPath);
this.log(`Removed ${oldPath} from cache`);
@ -1370,7 +1395,7 @@ export class TaskManager extends Component {
*/
private removeFileFromIndex(file: TFile): void {
// 使用 indexer 的方法来删除文件
this.indexer.updateIndexWithTasks(file.path, []);
this.indexer.cleanupFileCache(file.path);
// 从缓存中删除文件
try {
@ -2899,16 +2924,22 @@ export class TaskManager extends Component {
const {
clearProjectCaches = true,
preserveValidCaches = false,
logCacheStats = false
logCacheStats = false,
} = options || {};
this.log(`Force reindexing all tasks (clearProjectCaches: ${clearProjectCaches}, preserveValidCaches: ${preserveValidCaches})`);
this.log(
`Force reindexing all tasks (clearProjectCaches: ${clearProjectCaches}, preserveValidCaches: ${preserveValidCaches})`
);
// Log cache statistics before clearing if requested
if (logCacheStats && this.taskParsingService) {
try {
const beforeStats = this.taskParsingService.getDetailedCacheStats();
this.log("Cache statistics before clearing: " + JSON.stringify(beforeStats.summary, null, 2));
const beforeStats =
this.taskParsingService.getDetailedCacheStats();
this.log(
"Cache statistics before clearing: " +
JSON.stringify(beforeStats.summary, null, 2)
);
} catch (error) {
console.warn("Failed to get cache statistics:", error);
}
@ -2926,13 +2957,19 @@ export class TaskManager extends Component {
if (preserveValidCaches) {
// Smart cache clearing - only clear stale entries
if ((this.taskParsingService as any).projectConfigManager) {
const clearedCount = await (this.taskParsingService as any).projectConfigManager.clearStaleEntries();
this.log(`Smart cache clearing: removed ${clearedCount} stale entries`);
const clearedCount = await (
this.taskParsingService as any
).projectConfigManager.clearStaleEntries();
this.log(
`Smart cache clearing: removed ${clearedCount} stale entries`
);
}
} else {
// Full cache clearing (default behavior)
this.taskParsingService.clearAllCaches();
this.log("Cleared all project-related caches (config, data, metadata)");
this.log(
"Cleared all project-related caches (config, data, metadata)"
);
}
} catch (error) {
console.error("Error clearing project caches:", error);
@ -2987,8 +3024,12 @@ export class TaskManager extends Component {
// Log cache statistics after rebuilding if requested
if (logCacheStats && this.taskParsingService) {
try {
const afterStats = this.taskParsingService.getDetailedCacheStats();
this.log("Cache statistics after rebuilding: " + JSON.stringify(afterStats.summary, null, 2));
const afterStats =
this.taskParsingService.getDetailedCacheStats();
this.log(
"Cache statistics after rebuilding: " +
JSON.stringify(afterStats.summary, null, 2)
);
} catch (error) {
console.warn("Failed to get final cache statistics:", error);
}

View file

@ -98,6 +98,8 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
onCompletion: new Map<string, Set<string>>(),
dependsOn: new Map<string, Set<string>>(),
taskId: new Map<string, Set<string>>(),
fileMtimes: new Map<string, number>(),
fileProcessedTimes: new Map<string, number>(),
};
}
@ -238,7 +240,11 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
* Update the index with tasks parsed by external components
* This is the primary method for updating the index
*/
public updateIndexWithTasks(filePath: string, tasks: Task[]): void {
public updateIndexWithTasks(
filePath: string,
tasks: Task[],
fileMtime?: number
): void {
// Remove existing tasks for this file first
this.removeFileFromIndex(filePath);
@ -257,6 +263,11 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
// Update file index
this.taskCache.files.set(filePath, fileTaskIds);
this.lastIndexTime.set(filePath, Date.now());
// Update file mtime if provided
if (fileMtime !== undefined) {
this.updateFileMtime(filePath, fileMtime);
}
}
/**
@ -1052,6 +1063,75 @@ export class TaskIndexer extends Component implements TaskIndexerInterface {
this.taskCache = this.initEmptyCache();
}
/**
* Check if a file has changed since last processing
*/
public isFileChanged(filePath: string, currentMtime: number): boolean {
const lastMtime = this.taskCache.fileMtimes.get(filePath);
return lastMtime === undefined || lastMtime < currentMtime;
}
/**
* Get the last known modification time for a file
*/
public getFileLastMtime(filePath: string): number | undefined {
return this.taskCache.fileMtimes.get(filePath);
}
/**
* Update the modification time for a file
*/
public updateFileMtime(filePath: string, mtime: number): void {
this.taskCache.fileMtimes.set(filePath, mtime);
this.taskCache.fileProcessedTimes.set(filePath, Date.now());
}
/**
* Check if we have valid cache for a file
*/
public hasValidCache(filePath: string, currentMtime: number): boolean {
// Check if file has tasks in cache
const hasTasksInCache = this.taskCache.files.has(filePath);
// Check if file hasn't changed
const hasNotChanged = !this.isFileChanged(filePath, currentMtime);
return hasTasksInCache && hasNotChanged;
}
/**
* Clean up cache for a specific file
*/
public cleanupFileCache(filePath: string): void {
// Remove from file mtime cache
this.taskCache.fileMtimes.delete(filePath);
this.taskCache.fileProcessedTimes.delete(filePath);
// Remove from other caches (handled by existing removeFileFromIndex)
this.removeFileFromIndex(filePath);
}
/**
* Validate cache consistency and fix any issues
*/
public validateCacheConsistency(): void {
// Check for files in mtime cache but not in file index
for (const filePath of this.taskCache.fileMtimes.keys()) {
if (!this.taskCache.files.has(filePath)) {
this.taskCache.fileMtimes.delete(filePath);
this.taskCache.fileProcessedTimes.delete(filePath);
}
}
// Check for files in file index but not in mtime cache
for (const filePath of this.taskCache.files.keys()) {
if (!this.taskCache.fileMtimes.has(filePath)) {
// This is acceptable - mtime might not be set for older cache entries
// We don't need to remove the file from index
}
}
}
/**
* Set the cache from an external source (e.g. persisted cache)
*/

View file

@ -18,7 +18,10 @@ import {
TaskParseResult,
} from "./TaskIndexWorkerMessage";
import { FileMetadataTaskParser } from "./FileMetadataTaskParser";
import { FileParsingConfiguration, FileMetadataInheritanceConfig } from "../../common/setting-definition";
import {
FileParsingConfiguration,
FileMetadataInheritanceConfig,
} from "../../common/setting-definition";
// Import worker and utilities
// @ts-ignore Ignore type error for worker import
@ -109,6 +112,8 @@ interface TaskMetadata {
mtime: number;
size: number;
};
/** Whether this metadata came from cache */
fromCache?: boolean;
}
/**
@ -155,6 +160,14 @@ export class TaskWorkerManager extends Component {
private fileMetadataParser?: FileMetadataTaskParser;
/** Whether workers have been initialized to prevent multiple initialization */
private initialized: boolean = false;
/** Reference to task indexer for cache checking */
private taskIndexer?: any;
/** Performance statistics */
private stats = {
filesSkipped: 0,
filesProcessed: 0,
cacheHitRatio: 0,
};
/**
* Create a new worker pool
@ -304,6 +317,73 @@ export class TaskWorkerManager extends Component {
return worker;
}
/**
* Set the task indexer reference for cache checking
*/
public setTaskIndexer(taskIndexer: any): void {
this.taskIndexer = taskIndexer;
}
/**
* Update cache hit ratio statistics
*/
private updateCacheHitRatio(): void {
const totalFiles = this.stats.filesSkipped + this.stats.filesProcessed;
this.stats.cacheHitRatio =
totalFiles > 0 ? this.stats.filesSkipped / totalFiles : 0;
}
/**
* Get performance statistics
*/
public getStats() {
return { ...this.stats };
}
/**
* Check if a file should be processed (not in valid cache)
*/
private shouldProcessFile(file: TFile): boolean {
if (!this.taskIndexer) {
return true; // No indexer, always process
}
// Check if mtime optimization is enabled
if (
!this.options.settings?.fileParsingConfig?.enableMtimeOptimization
) {
return true; // Optimization disabled, always process
}
return !this.taskIndexer.hasValidCache(file.path, file.stat.mtime);
}
/**
* Get cached tasks for a file if available
*/
private getCachedTasksForFile(filePath: string): Task[] | null {
if (!this.taskIndexer) {
return null;
}
const taskIds = this.taskIndexer.getCache().files.get(filePath);
if (!taskIds) {
return null;
}
const tasks: Task[] = [];
const taskCache = this.taskIndexer.getCache().tasks;
for (const taskId of taskIds) {
const task = taskCache.get(taskId);
if (task) {
tasks.push(task);
}
}
return tasks.length > 0 ? tasks : null;
}
/**
* Process a single file for tasks
*/
@ -315,6 +395,19 @@ export class TaskWorkerManager extends Component {
let existing = this.outstanding.get(file.path);
if (existing) return existing;
// Check if we can use cached results
if (!this.shouldProcessFile(file)) {
const cachedTasks = this.getCachedTasksForFile(file.path);
if (cachedTasks) {
this.stats.filesSkipped++;
this.updateCacheHitRatio();
this.log(
`Using cached tasks for ${file.path} (${cachedTasks.length} tasks)`
);
return Promise.resolve(cachedTasks);
}
}
let promise = deferred<Task[]>();
this.outstanding.set(file.path, promise);
@ -354,14 +447,44 @@ export class TaskWorkerManager extends Component {
return new Map<string, Task[]>();
}
// Pre-filter files: separate cached from uncached
const filesToProcess: TFile[] = [];
const resultMap = new Map<string, Task[]>();
let cachedCount = 0;
for (const file of files) {
if (!this.shouldProcessFile(file)) {
const cachedTasks = this.getCachedTasksForFile(file.path);
if (cachedTasks) {
resultMap.set(file.path, cachedTasks);
cachedCount++;
continue;
}
}
filesToProcess.push(file);
}
this.log(
`Batch processing: ${cachedCount} files from cache, ${
filesToProcess.length
} files to process (cache hit ratio: ${
cachedCount > 0
? ((cachedCount / files.length) * 100).toFixed(1)
: 0
}%)`
);
if (filesToProcess.length === 0) {
return resultMap; // All files were cached
}
this.isProcessingBatch = true;
this.processedFiles = 0;
this.totalFilesToProcess = files.length;
this.totalFilesToProcess = filesToProcess.length;
this.log(`Processing batch of ${files.length} files`);
// 创建一个结果映射
const resultMap = new Map<string, Task[]>();
this.log(
`Processing batch of ${filesToProcess.length} files (${cachedCount} cached)`
);
try {
// 将文件分成更小的批次,避免一次性提交太多任务
@ -397,8 +520,8 @@ export class TaskWorkerManager extends Component {
}
};
for (let i = 0; i < files.length; i += batchSize) {
const subBatch = files.slice(i, i + batchSize);
for (let i = 0; i < filesToProcess.length; i += batchSize) {
const subBatch = filesToProcess.slice(i, i + batchSize);
// 为子批次创建处理任务并添加到队列
processingQueue.push(async () => {
@ -450,7 +573,9 @@ export class TaskWorkerManager extends Component {
console.error("Error during batch processing:", error);
} finally {
this.isProcessingBatch = false;
this.log(`Completed batch processing of ${files.length} files`);
this.log(
`Completed batch processing of ${files.length} files (${cachedCount} from cache, ${filesToProcess.length} processed)`
);
}
return resultMap;
@ -676,6 +801,10 @@ export class TaskWorkerManager extends Component {
promise.resolve(allTasks);
this.outstanding.delete(file.path);
// Update statistics
this.stats.filesProcessed++;
this.updateCacheHitRatio();
} else if (data.type === "batchResult") {
// For batch results, we handle differently as we don't have tasks directly
promise.reject(