diff --git a/package-lock.json b/package-lock.json index f4af2aee..83fb50a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "@radix-ui/react-dropdown-menu": "^2.1.4", "@radix-ui/react-label": "^2.1.0", "@radix-ui/react-popover": "^1.1.4", + "@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-scroll-area": "^1.2.9", "@radix-ui/react-select": "^2.1.2", "@radix-ui/react-separator": "^1.1.7", @@ -6329,6 +6330,68 @@ } } }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.7.tgz", + "integrity": "sha512-vPdg/tF6YC/ynuBIJlk1mm7Le0VgW6ub6J2UWnTQ7/D23KXcPI1qy+0vBkgKgd38RCMJavBXpB83HPNFMTb0Fg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.1.2", + "@radix-ui/react-primitive": "2.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress/node_modules/@radix-ui/react-context": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.2.tgz", + "integrity": "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress/node_modules/@radix-ui/react-primitive": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.3.tgz", + "integrity": "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.2.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-roving-focus": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.1.tgz", diff --git a/package.json b/package.json index b718f6ca..f4ca83b9 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "@radix-ui/react-dropdown-menu": "^2.1.4", "@radix-ui/react-label": "^2.1.0", "@radix-ui/react-popover": "^1.1.4", + "@radix-ui/react-progress": "^1.1.7", "@radix-ui/react-scroll-area": "^1.2.9", "@radix-ui/react-select": "^2.1.2", "@radix-ui/react-separator": "^1.1.7", diff --git a/src/LLMProviders/projectLoadTracker.ts b/src/LLMProviders/projectLoadTracker.ts new file mode 100644 index 00000000..cb48296e --- /dev/null +++ b/src/LLMProviders/projectLoadTracker.ts @@ -0,0 +1,279 @@ +import { + FailedItem, + ProjectConfig, + projectContextLoadAtom, + setProjectContextLoadState, + updateProjectContextLoadState, +} from "@/aiParams"; +import { ContextCache } from "@/cache/projectContextCache"; +import { logInfo } from "@/logger"; +import { settingsStore } from "@/settings/model"; +import { err2String } from "@/utils"; +import { App, TFile } from "obsidian"; +import { isRateLimitError } from "@/utils/rateLimitUtils"; + +/** + * ProjectLoadTracker is responsible for managing the progress tracking of project file processing + */ +export class ProjectLoadTracker { + private static instance: ProjectLoadTracker; + private app: App; + + private constructor(app: App) { + this.app = app; + } + + public static getInstance(app: App): ProjectLoadTracker { + if (!ProjectLoadTracker.instance) { + ProjectLoadTracker.instance = new ProjectLoadTracker(app); + } + return ProjectLoadTracker.instance; + } + + /** + * Clear all project context loading states + */ + public clearAllLoadStates(): void { + setProjectContextLoadState({ + success: [], + failed: [], + processingFiles: [], + total: [], + }); + } + + /** + * Wrap an operation and track its execution status + */ + public async executeWithProcessTracking( + key: string, + type: FailedItem["type"], + operation: () => Promise + ): Promise { + this.setFileOrUrlStartProcess(key); + try { + const result = await operation(); + this.setFileOrUrlProcessSuccessful(key); + return result; + } catch (error) { + const errorMessage = isRateLimitError(error) + ? "Rate limit exceeded. (Rate limit: 50 files or 100MB per 3 hours, whichever is reached first)" + : err2String(error); + + this.setFileOrUrlProcessFailed(key, type, errorMessage); + throw error; // throw error to outer layer + } + } + + /** + * Mark a file or URL as processing started + */ + private setFileOrUrlStartProcess(key: string): void { + settingsStore.set(projectContextLoadAtom, (prev) => { + const newState = { ...prev }; + + // note: we remove the failed file from the failed list when it starts processing + if (newState.failed.find((item) => item.path === key)) { + newState.failed = newState.failed.filter((file) => file.path !== key); + } + + // note: we remove the success file from the success list when it starts processing + // For the case where the file cacheKey still exists, but the actual cached content is missing + if (newState.success.includes(key)) { + newState.success = newState.success.filter((file) => file !== key); + } + + // Add to processing files list + if (!newState.processingFiles.includes(key)) { + newState.processingFiles = [...newState.processingFiles, key]; + } + + // Ensure file is in the total list + if (!newState.total.includes(key)) { + newState.total = [...newState.total, key]; + } + + return newState; + }); + } + + /** + * Mark a process as successful + */ + private setFileOrUrlProcessSuccessful(key: string): void { + updateProjectContextLoadState("processingFiles", (prev) => prev.filter((file) => file !== key)); + updateProjectContextLoadState("success", (prev) => { + if (!prev.includes(key)) { + return [...prev, key]; + } + return prev; + }); + } + + /** + * Mark a process as failed + */ + private setFileOrUrlProcessFailed(key: string, type: FailedItem["type"], error?: string): void { + updateProjectContextLoadState("processingFiles", (prev) => prev.filter((file) => file !== key)); + + updateProjectContextLoadState("failed", (prev) => { + const existingFailed = prev.find((item) => item.path === key); + if (!existingFailed) { + const failedItem: FailedItem = { + path: key, + type, + error, + timestamp: Date.now(), + }; + return [...prev, failedItem]; + } + return prev; + }); + } + + /** + * Pre-compute all items that need to be processed in the project + */ + public preComputeAllItems(project: ProjectConfig, projectAllFiles: TFile[]): void { + logInfo(`[preComputeAllItems] Starting pre-computation for project: ${project.name}`); + + const allItems: string[] = []; + + // 1. Count all matching files (markdown and non-markdown) + // Add all matching file paths to the list + allItems.push(...projectAllFiles.map((file: TFile) => file.path)); + + // 2. Count all Web URLs + const configuredWebUrls = project.contextSource?.webUrls?.trim() || ""; + if (configuredWebUrls) { + const webUrls = configuredWebUrls.split("\n").filter((url) => url.trim()); + allItems.push(...webUrls); + } + + // 3. Count all YouTube URLs + const configuredYoutubeUrls = project.contextSource?.youtubeUrls?.trim() || ""; + if (configuredYoutubeUrls) { + const youtubeUrls = configuredYoutubeUrls.split("\n").filter((url) => url.trim()); + allItems.push(...youtubeUrls); + } + + // Add all items to the total list + if (allItems.length > 0) { + const uniqueItems = [...new Set([...allItems])]; + updateProjectContextLoadState("total", (_) => uniqueItems); + logInfo( + `[preComputeAllItems] Project ${project.name}: Added ${allItems.length} items to tracking (${uniqueItems.length} total unique items)` + ); + } + } + + /** + * Mark all cached items(besides Non-markdown files) as successful + */ + public markAllCachedItemsAsSuccess( + project: ProjectConfig, + contextCache: ContextCache, + projectAllFiles: TFile[] + ): void { + logInfo(`[markAllCachedItemsAsSuccess] Starting for project: ${project.name || "default"}`); + + // 1. Mark cached Web URLs + const configuredWebUrls = project.contextSource?.webUrls?.trim() || ""; + if (configuredWebUrls) { + const urlsInConfig = configuredWebUrls.split("\n").filter((url) => url.trim()); + const cachedUrls = urlsInConfig.filter((url) => contextCache.webContexts[url]); + cachedUrls.forEach((url) => { + this.markCachedItemAsSuccess(url); + }); + if (cachedUrls.length > 0) { + logInfo( + `[markAllCachedItemsAsSuccess] Project ${project.name}: Marked ${cachedUrls.length} cached Web URLs as successful` + ); + } + } + + // 2. Mark cached YouTube URLs + const configuredYoutubeUrls = project.contextSource?.youtubeUrls?.trim() || ""; + if (configuredYoutubeUrls) { + const urlsInConfig = configuredYoutubeUrls.split("\n").filter((url) => url.trim()); + const cachedUrls = urlsInConfig.filter((url) => contextCache.youtubeContexts[url]); + cachedUrls.forEach((url) => { + this.markCachedItemAsSuccess(url); + }); + if (cachedUrls.length > 0) { + logInfo( + `[markAllCachedItemsAsSuccess] Project ${project.name}: Marked ${cachedUrls.length} cached YouTube URLs as successful` + ); + } + } + + // 3. Only mark markdown files present in fileContexts as successful, does not include Non-markdown files. + // because track Non-markdown in the processNonMarkdownFiles method + if (contextCache.fileContexts) { + // only for markdown files + const matchingFilesSet = new Set( + projectAllFiles.filter((file) => file.extension === "md").map((file: TFile) => file.path) + ); + + const cachedFilesToMark = Object.keys(contextCache.fileContexts).filter((filePath) => + matchingFilesSet.has(filePath) + ); + + cachedFilesToMark.forEach((filePath) => { + this.markCachedItemAsSuccess(filePath); + }); + + if (cachedFilesToMark.length > 0) { + logInfo( + `[markAllCachedItemsAsSuccess] Project ${project.name}: Marked ${ + cachedFilesToMark.length + } cached files that match current project patterns as successful.` + ); + } + } + } + + /** + * Mark a cached item as successful + */ + public markCachedItemAsSuccess(key: string): void { + updateProjectContextLoadState("total", (prev) => { + if (!prev.includes(key)) { + return [...prev, key]; + } + return prev; + }); + + // Mark as successful directly + updateProjectContextLoadState("success", (prev) => { + if (!prev.includes(key)) { + return [...prev, key]; + } + return prev; + }); + } + + public makeItemFailed(key: string, type: FailedItem["type"], error?: string): void { + updateProjectContextLoadState("total", (prev) => { + if (!prev.includes(key)) { + return [...prev, key]; + } + return prev; + }); + + // Check if this item is already in the failed list + updateProjectContextLoadState("failed", (prev) => { + const existingFailed = prev.find((item) => item.path === key); + if (!existingFailed) { + const failedItem: FailedItem = { + path: key, + type, + error, + timestamp: Date.now(), + }; + return [...prev, failedItem]; + } + return prev; + }); + } +} diff --git a/src/LLMProviders/projectManager.ts b/src/LLMProviders/projectManager.ts index b9eff45a..e0ff2d06 100644 --- a/src/LLMProviders/projectManager.ts +++ b/src/LLMProviders/projectManager.ts @@ -1,4 +1,5 @@ import { + FailedItem, getChainType, isProjectMode, ProjectConfig, @@ -23,6 +24,7 @@ import { App, Notice, TFile } from "obsidian"; import VectorStoreManager from "../search/vectorStoreManager"; import { BrevilabsClient } from "./brevilabsClient"; import ChainManager from "./chainManager"; +import { ProjectLoadTracker } from "./projectLoadTracker"; export default class ProjectManager { public static instance: ProjectManager; @@ -32,6 +34,7 @@ export default class ProjectManager { private readonly chainMangerInstance: ChainManager; private readonly projectContextCache: ProjectContextCache; private fileParserManager: FileParserManager; + private loadTracker: ProjectLoadTracker; private constructor(app: App, vectorStoreManager: VectorStoreManager, plugin: CopilotPlugin) { this.app = app; @@ -45,6 +48,7 @@ export default class ProjectManager { true, null ); + this.loadTracker = ProjectLoadTracker.getInstance(this.app); // Set up subscriptions subscribeToModelKeyChange(async () => { @@ -124,6 +128,8 @@ export default class ProjectManager { public async switchProject(project: ProjectConfig | null): Promise { try { + // Clear all project context loading states + this.loadTracker.clearAllLoadStates(); setProjectLoading(true); logInfo("Project loading started..."); @@ -187,6 +193,10 @@ export default class ProjectManager { } private async loadProjectContext(project: ProjectConfig): Promise { + // for update context condition + this.loadTracker.clearAllLoadStates(); + setProjectLoading(true); + try { if (!project.contextSource) { logWarn(`[loadProjectContext] Project ${project.name}: No contextSource. Aborting.`); @@ -194,86 +204,31 @@ export default class ProjectManager { } logInfo(`[loadProjectContext] Starting for project: ${project.name}`); - const initialProjectCache = await this.projectContextCache.get(project); - const contextCache = initialProjectCache || { - markdownContext: "", - webContexts: {}, - youtubeContexts: {}, - fileContexts: {}, - timestamp: Date.now(), - markdownNeedsReload: true, - }; - if (!initialProjectCache) { - logInfo( - `[loadProjectContext] Project ${project.name}: No existing cache found, building fresh context.` - ); - } else { - logInfo( - `[loadProjectContext] Project ${project.name}: Existing cache found. MarkdownNeedsReload: ${contextCache.markdownNeedsReload}` - ); - } + logInfo( + `[loadProjectContext] Project ${project.name}: Cleared all project context load states` + ); + + const contextCache = await this.projectContextCache.getOrInitializeCache(project); + + const projectAllFiles = this.getProjectAllFiles(project); + + // Pre-count all items that need to be processed + this.loadTracker.preComputeAllItems(project, projectAllFiles); + this.loadTracker.markAllCachedItemsAsSuccess(project, contextCache, projectAllFiles); const [updatedContextCacheAfterSources] = await Promise.all([ - this.processMarkdownFiles(project, contextCache), + this.processMarkdownFiles(project, contextCache, projectAllFiles), this.processWebUrls(project, contextCache), this.processYoutubeUrls(project, contextCache), ]); - // After other contexts are processed, ensure all referenced non-markdown files are parsed and cached - if (updatedContextCacheAfterSources.fileContexts) { - const fileContextCount = Object.keys(updatedContextCacheAfterSources.fileContexts).length; - logInfo( - `[loadProjectContext] Project ${project.name}: Checking ${fileContextCount} fileContexts for non-markdown processing.` - ); - - if (fileContextCount > 0) { - this.fileParserManager = new FileParserManager( - BrevilabsClient.getInstance(), - this.app.vault, - true, - project - ); - let processedNonMdCount = 0; - for (const filePath in updatedContextCacheAfterSources.fileContexts) { - const file = this.app.vault.getAbstractFileByPath(filePath); - if (file instanceof TFile && file.extension !== "md") { - if (this.fileParserManager.supportsExtension(file.extension)) { - try { - const existingContent = await this.projectContextCache.getFileContext( - project, - filePath - ); - if (!existingContent) { - logInfo( - `[loadProjectContext] Project ${project.name}: Parsing/caching new/updated file: ${filePath}` - ); - await this.fileParserManager.parseFile(file, this.app.vault); - processedNonMdCount++; - } // else { logInfo for skipped can be too verbose } - } catch (error) { - logError( - `[loadProjectContext] Project ${project.name}: Error parsing file ${filePath}:`, - error - ); - - // Check if this is a rate limit error and re-throw it to fail the entire operation - if (isRateLimitError(error)) { - throw error; // Re-throw to fail the entire operation - } - } - } - } - } - if (processedNonMdCount > 0) { - logInfo( - `[loadProjectContext] Project ${project.name}: Processed and cached ${processedNonMdCount} non-markdown files.` - ); - } - } - } - updatedContextCacheAfterSources.timestamp = Date.now(); - await this.projectContextCache.set(project, updatedContextCacheAfterSources); + // Note: Since non-markdown files cannot pass cache parameters , so we need to save the context cache first + await this.projectContextCache.setCacheSafely(project, updatedContextCacheAfterSources); + + // After other contexts are processed, ensure all referenced non-markdown files are parsed and cached + await this.processNonMarkdownFiles(project, projectAllFiles); + logInfo(`[loadProjectContext] Completed for project: ${project.name}.`); return updatedContextCacheAfterSources; } catch (error) { @@ -315,11 +270,10 @@ export default class ProjectManager { const nextUrls = nextWebUrls.split("\n").filter((url) => url.trim()); // Remove context for URLs that no longer exist - for (const url of prevUrls) { - if (!nextUrls.includes(url)) { - await this.projectContextCache.removeWebUrl(nextProject, url); - } - } + await this.projectContextCache.removeWebUrls( + nextProject, + prevUrls.filter((url) => !nextUrls.includes(url)) + ); } // Check if YouTube URLs configuration has changed @@ -332,11 +286,10 @@ export default class ProjectManager { const nextUrls = nextYoutubeUrls.split("\n").filter((url) => url.trim()); // Remove context for URLs that no longer exist - for (const url of prevUrls) { - if (!nextUrls.includes(url)) { - await this.projectContextCache.removeYoutubeUrl(nextProject, url); - } - } + await this.projectContextCache.removeYoutubeUrls( + nextProject, + prevUrls.filter((url) => !nextUrls.includes(url)) + ); } } catch (error) { logError(`Error comparing project configurations: ${error}`); @@ -424,7 +377,7 @@ export default class ProjectManager { // Retrieve file content from FileCache const content = - (await this.projectContextCache.getFileContext(project, filePath)) || + (await this.projectContextCache.getOrReuseFileContext(project, filePath)) || "[Content not available]"; // This is expected for files not processed into FileCache return `[[${fileName}]]\npath: ${filePath}\ntype: ${fileType}\nmodified: ${new Date(fileContext.timestamp).toISOString()}\n\n${content}`; @@ -497,82 +450,50 @@ ${contextParts.join("\n\n")} private async processMarkdownFiles( project: ProjectConfig, - contextCache: ContextCache + contextCache: ContextCache, + projectAllFiles: TFile[] ): Promise { logInfo(`[processMarkdownFiles] Starting for project: ${project.name}`); - const initialFileContextsCount = Object.keys(contextCache.fileContexts || {}).length; - if (project.contextSource?.inclusions || project.contextSource?.exclusions) { - contextCache = await this.projectContextCache.updateProjectFilesFromPatterns( + if ( + contextCache.markdownNeedsReload || + !contextCache.markdownContext || + !contextCache.markdownContext.trim() + ) { + logInfo(`[processMarkdownFiles] Project ${project.name}: Processing markdown content.`); + const markdownContent = await this.processMarkdownFileContext(projectAllFiles); + + // add context reference to markdown file + this.projectContextCache.updateProjectMarkdownFilesFromPatterns( project, - contextCache + contextCache, + projectAllFiles ); - const newFileContextsCount = Object.keys(contextCache.fileContexts || {}).length; - if (newFileContextsCount > initialFileContextsCount) { - logInfo( - `[processMarkdownFiles] Project ${project.name}: Added ${newFileContextsCount - initialFileContextsCount} new file references via updateProjectFilesFromPatterns.` - ); - } - if ( - contextCache.markdownNeedsReload || - !contextCache.markdownContext || - !contextCache.markdownContext.trim() - ) { - logInfo(`[processMarkdownFiles] Project ${project.name}: Processing markdown content.`); - const markdownContent = await this.processFileContext( - project.contextSource.inclusions, - project.contextSource.exclusions, - project - ); - contextCache.markdownContext = markdownContent; - contextCache.markdownNeedsReload = false; - logInfo(`[processMarkdownFiles] Project ${project.name}: Markdown content updated.`); - } else { - logInfo( - `[processMarkdownFiles] Project ${project.name}: Markdown content already up-to-date.` - ); - } + contextCache.markdownContext = markdownContent; + contextCache.markdownNeedsReload = false; + + logInfo(`[processMarkdownFiles] Project ${project.name}: Markdown content updated.`); + } else { + logInfo( + `[processMarkdownFiles] Project ${project.name}: Markdown content already up-to-date.` + ); } + logInfo( `[processMarkdownFiles] Completed for project: ${project.name}. Total fileContexts: ${Object.keys(contextCache.fileContexts || {}).length}` ); return contextCache; } - private async processFileContext( - inclusions?: string, - exclusions?: string, - project?: ProjectConfig - ): Promise { - if (!inclusions && !exclusions) { - return ""; - } - - if (!project) { - return ""; - } - - // NOTE: Must not fallback to GLOBAL inclusions and exclusions in Copilot settings in Projects! - // This is to avoid project inclusions in the project that conflict with the global ones - // Project UI should be the ONLY source of truth for project inclusions and exclusions - const { inclusions: inclusionPatterns, exclusions: exclusionPatterns } = getMatchingPatterns({ - inclusions, - exclusions, - isProject: true, - }); - + private async processMarkdownFileContext(projectAllFiles: TFile[]): Promise { // FileParserManager will be used to process these files when they're accessed, // either immediately or on-demand when the context is formatted // Get all markdown files that match the inclusion/exclusion patterns // Note: We're only processing markdown files here, other file types // are handled by FileParserManager and stored in the file cache - const files = this.app.vault.getFiles().filter((file) => { - return ( - file.extension === "md" && shouldIndexFile(file, inclusionPatterns, exclusionPatterns, true) - ); - }); + const files = projectAllFiles.filter((file) => file.extension === "md"); logInfo(`Found ${files.length} markdown files to process for project context`); @@ -583,15 +504,25 @@ ${contextParts.join("\n\n")} let metadata = ""; try { - const stat = await this.app.vault.adapter.stat(file.path); + // Only process markdown files here + const [stat, fileContent] = await this.loadTracker.executeWithProcessTracking( + file.path, + "md", + async () => { + return Promise.all([ + this.app.vault.adapter.stat(file.path), + this.app.vault.read(file), + ]); + } + ); + metadata = `[[${file.basename}]] path: ${file.path} type: ${file.extension} created: ${stat ? new Date(stat.ctime).toISOString() : "unknown"} modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`; - // Only process markdown files here - content = await this.app.vault.read(file); + content = fileContent; logInfo(`Completed processing markdown file: ${file.path}`); } catch (error) { logError(`Error processing file ${file.path}: ${error}`); @@ -650,8 +581,8 @@ modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`; } const webContextPromises = urlsToFetch.map(async (url) => { - // processWebUrlsContext itself should log errors if a specific URL fetch fails. - const webContext = await this.processWebUrlsContext(url); + // processWebUrlContext itself should log errors if a specific URL fetch fails. + const webContext = await this.processWebUrlContext(url); if (webContext) { logInfo( `[processWebUrls] Project ${project.name}: Successfully fetched content for URL: ${url.substring(0, 50)}...` @@ -717,7 +648,7 @@ modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`; } const youtubeContextPromises = urlsToFetch.map(async (url) => { - const youtubeContext = await this.processYoutubeUrlsContext(url); + const youtubeContext = await this.processYoutubeUrlContext(url); if (youtubeContext) { logInfo( `[processYoutubeUrls] Project ${project.name}: Successfully fetched transcript for YouTube URL: ${url.substring(0, 50)}...` @@ -742,44 +673,252 @@ modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`; return contextCache; } - private async processWebUrlsContext(webUrls?: string): Promise { - if (!webUrls?.trim()) { + private async processWebUrlContext(webUrl?: string): Promise { + if (!webUrl?.trim()) { return ""; } try { const mention = Mention.getInstance(); - const { urlContext } = await mention.processUrls(webUrls); + const { urlContext } = await this.loadTracker.executeWithProcessTracking( + webUrl, + "web", + async () => { + const result = await mention.processUrls(webUrl); + + if (result.processedErrorUrls[webUrl]) { + throw new Error(result.processedErrorUrls[webUrl]); + } + return result; + } + ); return urlContext || ""; } catch (error) { - logError(`Failed to process web URLs: ${error}`); - new Notice(`Failed to process web URLs: ${err2String(error)}`); + logError(`Failed to process web URL: ${error}`); return ""; } } - private async processYoutubeUrlsContext(youtubeUrls?: string): Promise { - if (!youtubeUrls?.trim()) { + private async processYoutubeUrlContext(youtubeUrl?: string): Promise { + if (!youtubeUrl?.trim()) { return ""; } - const urls = youtubeUrls.split("\n").filter((url) => url.trim()); - const processPromises = urls.map(async (url) => { - try { - const response = await BrevilabsClient.getInstance().youtube4llm(url); - if (response.response.transcript) { - return `\n\nYouTube transcript from ${url}:\n${response.response.transcript}`; + try { + const response = await this.loadTracker.executeWithProcessTracking( + youtubeUrl, + "youtube", + async () => { + return BrevilabsClient.getInstance().youtube4llm(youtubeUrl); } - return ""; - } catch (error) { - logError(`Failed to process YouTube URL ${url}: ${error}`); - new Notice(`Failed to process YouTube URL ${url}: ${err2String(error)}`); - return ""; + ); + if (response.response.transcript) { + return `\n\nYouTube transcript from ${youtubeUrl}:\n${response.response.transcript}`; } + return ""; + } catch (error) { + logError(`Failed to process YouTube URL ${youtubeUrl}: ${error}`); + new Notice(`Failed to process YouTube URL ${youtubeUrl}: ${err2String(error)}`); + return ""; + } + } + + private async processNonMarkdownFiles( + project: ProjectConfig, + projectAllFiles: TFile[] + ): Promise { + const nonMarkdownFiles = projectAllFiles.filter((file) => file.extension !== "md"); + + logInfo( + `[loadProjectContext] Project ${project.name}: Checking for non-markdown processing: ${nonMarkdownFiles.length} files .` + ); + + if (nonMarkdownFiles.length <= 0) { + return; + } + + this.fileParserManager = new FileParserManager( + BrevilabsClient.getInstance(), + this.app.vault, + true, + project + ); + + let processedNonMdCount = 0; + + for (const file of nonMarkdownFiles) { + const filePath = file.path; + if (this.fileParserManager.supportsExtension(file.extension)) { + try { + await this.loadTracker.executeWithProcessTracking(filePath, "nonMd", async () => { + const existingContent = await this.projectContextCache.getOrReuseFileContext( + project, + filePath + ); + if (existingContent) { + processedNonMdCount++; + } else { + logInfo( + `[loadProjectContext] Project ${project.name}: Parsing/caching new/updated file: ${filePath}` + ); + + await this.fileParserManager.parseFile(file, this.app.vault); + processedNonMdCount++; + } + }); + } catch (error) { + logError( + `[loadProjectContext] Project ${project.name}: Error parsing file ${filePath}:`, + error + ); + + // Check if this is a rate limit error and re-throw it to fail the entire operation + if (isRateLimitError(error)) { + throw error; // Re-throw to fail the entire operation + } + } + } + } + + if (processedNonMdCount > 0) { + logInfo( + `[loadProjectContext] Project ${project.name}: Processed and cached ${processedNonMdCount} non-markdown files.` + ); + } + } + + /** + * Retry failed item + * @param failedItem Failed item information + */ + public async retryFailedItem(failedItem: FailedItem): Promise { + try { + if (!this.currentProjectId) { + logWarn("[retryFailedItem] No current project, aborting retry"); + return; + } + + const project = getSettings().projectList.find((p) => p.id === this.currentProjectId); + if (!project) { + logError(`[retryFailedItem] Current project not found: ${this.currentProjectId}`); + return; + } + + logInfo(`[retryFailedItem] Starting retry for ${failedItem.type} item: ${failedItem.path}`); + + // Handle different retry types + switch (failedItem.type) { + case "web": + await this.retryWebUrl(project, failedItem.path); + break; + case "youtube": + await this.retryYoutubeUrl(project, failedItem.path); + break; + case "md": + await this.retryMarkdownFile(project, failedItem.path); + break; + case "nonMd": + await this.retryNonMarkdownFile(project, failedItem.path); + break; + default: + logWarn(`[retryFailedItem] Unknown item type: ${failedItem.type}`); + return; + } + + logInfo(`[retryFailedItem] Successfully retried ${failedItem.type} item: ${failedItem.path}`); + new Notice(`Retry successful: ${failedItem.path}`); + } catch (error) { + logError( + `[retryFailedItem] Failed to retry ${failedItem.type} item ${failedItem.path}:`, + error + ); + new Notice(`Retry failed: ${err2String(error)}`); + } + } + + private async retryWebUrl(project: ProjectConfig, url: string): Promise { + const webContext = await this.processWebUrlContext(url); + if (!webContext) { + logWarn(`[retryWebUrl] Project ${project.name}: Fetched empty content for Web URL: ${url}`); + return; + } + + logInfo( + `[retryWebUrl] Project ${project.name}: Successfully fetched content for URL: ${url.substring(0, 50)}...` + ); + await this.projectContextCache.updateWebUrl(project, url, webContext); + } + + private async retryYoutubeUrl(project: ProjectConfig, url: string): Promise { + const youtubeContext = await this.processYoutubeUrlContext(url); + if (!youtubeContext) { + logWarn( + `[retryYoutubeUrl] Project ${project.name}: Fetched empty transcript for YouTube URL: ${url}` + ); + return; + } + + logInfo( + `[retryYoutubeUrl] Project ${project.name}: Successfully fetched transcript for YouTube URL: ${url.substring(0, 50)}...` + ); + await this.projectContextCache.updateYoutubeUrl(project, url, youtubeContext); + } + + private async retryMarkdownFile(project: ProjectConfig, filePath: string): Promise { + const file = this.app.vault.getAbstractFileByPath(filePath); + if (!(file instanceof TFile) || file.extension !== "md") { + throw new Error(`File not found or not a markdown file: ${filePath}`); + } + + try { + // add flag to track reprocessing of Markdown + await this.loadTracker.executeWithProcessTracking(file.path, "md", async () => {}); + + logInfo(`[retryMarkdownFile] Successfully reprocessed markdown file: ${filePath}`); + + // flag the markdown context as needing a reload + await this.projectContextCache.invalidateMarkdownContext(project); + } catch (error) { + logError(`[retryMarkdownFile] Error processing file ${filePath}: ${error}`); + throw error; + } + } + + private async retryNonMarkdownFile(project: ProjectConfig, filePath: string): Promise { + const file = this.app.vault.getAbstractFileByPath(filePath); + if (!(file instanceof TFile) || file.extension === "md") { + throw new Error(`File not found or is a markdown file: ${filePath}`); + } + + if (!this.fileParserManager.supportsExtension(file.extension)) { + throw new Error(`Unsupported file extension: ${file.extension}`); + } + + try { + await this.loadTracker.executeWithProcessTracking(filePath, "nonMd", async () => { + return this.fileParserManager.parseFile(file, this.app.vault); + }); + + logInfo(`[retryNonMarkdownFile] Successfully reprocessed non-markdown file: ${filePath}`); + } catch (error) { + logError(`[retryNonMarkdownFile] Error processing file ${filePath}: ${error}`); + throw error; + } + } + + private getProjectAllFiles(project: ProjectConfig) { + // NOTE: Must not fallback to GLOBAL inclusions and exclusions in Copilot settings in Projects! + // This is to avoid project inclusions in the project that conflict with the global ones + // Project UI should be the ONLY source of truth for project inclusions and exclusions + const { inclusions: inclusionPatterns, exclusions: exclusionPatterns } = getMatchingPatterns({ + inclusions: project.contextSource.inclusions, + exclusions: project.contextSource.exclusions, + isProject: true, }); - const results = await Promise.all(processPromises); - return results.join(""); + return this.app.vault.getFiles().filter((file: TFile) => { + return shouldIndexFile(file, inclusionPatterns, exclusionPatterns, true); + }); } public onunload(): void { diff --git a/src/aiParams.ts b/src/aiParams.ts index 56bfddbd..9950af07 100644 --- a/src/aiParams.ts +++ b/src/aiParams.ts @@ -38,6 +38,27 @@ const chainTypeAtom = atom( const currentProjectAtom = atom(null); const projectLoadingAtom = atom(false); +export interface FailedItem { + path: string; + type: "md" | "web" | "youtube" | "nonMd"; + error?: string; + timestamp?: number; +} + +interface ProjectContextLoadState { + success: Array; + failed: Array; + processingFiles: Array; + total: Array; +} + +export const projectContextLoadAtom = atom({ + success: [], + failed: [], + processingFiles: [], + total: [], +}); + const selectedTextContextsAtom = atom([]); export interface ProjectConfig { @@ -51,7 +72,7 @@ export interface ProjectConfig { maxTokens?: number; }; contextSource: { - inclusions: string; + inclusions?: string; exclusions?: string; webUrls?: string; youtubeUrls?: string; @@ -239,3 +260,50 @@ export function useSelectedTextContexts() { store: settingsStore, }); } + +/** + * Gets the project context load state from the atom. + */ +export function getProjectContextLoadState(): Readonly { + return settingsStore.get(projectContextLoadAtom); +} + +/** + * Sets the project context load state in the atom. + */ +export function setProjectContextLoadState(state: ProjectContextLoadState) { + settingsStore.set(projectContextLoadAtom, state); +} + +/** + * Updates a specific field in the project context load state. + */ +export function updateProjectContextLoadState( + key: K, + valueFn: (prev: ProjectContextLoadState[K]) => ProjectContextLoadState[K] +) { + settingsStore.set(projectContextLoadAtom, (prev) => ({ + ...prev, + [key]: valueFn(prev[key]), + })); +} + +/** + * Subscribes to changes in the project context load state. + */ +export function subscribeToProjectContextLoadChange( + callback: (state: ProjectContextLoadState) => void +): () => void { + return settingsStore.sub(projectContextLoadAtom, () => { + callback(settingsStore.get(projectContextLoadAtom)); + }); +} + +/** + * Hook to get the project context load state from the atom. + */ +export function useProjectContextLoad() { + return useAtom(projectContextLoadAtom, { + store: settingsStore, + }); +} diff --git a/src/cache/projectContextCache.ts b/src/cache/projectContextCache.ts index 62b83e07..d0ccff6b 100644 --- a/src/cache/projectContextCache.ts +++ b/src/cache/projectContextCache.ts @@ -6,6 +6,7 @@ import { getSettings } from "@/settings/model"; import { MD5 } from "crypto-js"; import { TAbstractFile, TFile, Vault } from "obsidian"; import debounce from "lodash.debounce"; +import { Mutex } from "async-mutex"; export interface ContextCache { // Markdown context @@ -38,6 +39,8 @@ export class ProjectContextCache { private vault: Vault; private fileCache: FileCache; private static readonly DEBOUNCE_DELAY = 5000; // 5 seconds + private projectMutexMap: Map = new Map(); + private mutexCreationMutex: Mutex = new Mutex(); // Global lock to protect project mutex creation private constructor() { this.vault = app.vault; @@ -65,6 +68,9 @@ export class ProjectContextCache { this.vault.off("modify", this.handleFileEvent); this.vault.off("delete", this.handleFileEvent); this.vault.off("rename", this.handleFileEvent); + + // Clean up project mutexes + this.projectMutexMap.clear(); } private initializeEventListeners() { @@ -143,6 +149,32 @@ export class ProjectContextCache { return `${this.cacheDir}/${cacheKey}.json`; } + private async getOrCreateProjectMutex(project: ProjectConfig): Promise { + const projectId = project.id; + + // Quick check without lock for performance + const existingMutex = this.projectMutexMap.get(projectId); + if (existingMutex) { + return existingMutex; + } + + // Use global lock to ensure atomic creation + return await this.mutexCreationMutex.runExclusive(async () => { + // Double-check inside the lock + const mutex = this.projectMutexMap.get(projectId); + if (mutex) { + return mutex; + } + + // Create new mutex safely + const newMutex = new Mutex(); + this.projectMutexMap.set(projectId, newMutex); + logInfo(`Created new mutex for project: ${project.name} (ID: ${projectId})`); + + return newMutex; + }); + } + async get(project: ProjectConfig): Promise { try { const cacheKey = this.getCacheKey(project); @@ -171,6 +203,25 @@ export class ProjectContextCache { } } + async getOrInitializeCache(project: ProjectConfig): Promise { + const initialProjectCache = await this.get(project); + + if (initialProjectCache) { + logInfo( + `[getOrInitializeCache] Project ${project.name}: Existing cache found. MarkdownNeedsReload: ${initialProjectCache.markdownNeedsReload}` + ); + return initialProjectCache; + } + + logInfo( + `[getOrInitializeCache] Project ${project.name}: No existing cache found, building fresh context.` + ); + + const newCache = this.createEmptyCache(); + await this.setWithoutMutex(project, newCache); + return newCache; + } + getSync(project: ProjectConfig): ContextCache | null { try { const cacheKey = this.getCacheKey(project); @@ -187,18 +238,37 @@ export class ProjectContextCache { } } - async set(project: ProjectConfig, contextCache: ContextCache): Promise { + private async set(project: ProjectConfig, contextCache: ContextCache): Promise { + const mutex = await this.getOrCreateProjectMutex(project); + + if (mutex.isLocked()) { + logInfo(`Waiting for project cache lock for project: ${project.name}`); + } + + return await mutex.runExclusive(async () => { + logInfo(`Acquired cache lock for project: ${project.name}`); + return await this.setWithoutMutex(project, contextCache); + }); + } + + private async setWithoutMutex(project: ProjectConfig, contextCache: ContextCache): Promise { try { await this.ensureCacheDir(); const cacheKey = this.getCacheKey(project); const cachePath = this.getCachePath(cacheKey); logInfo("Caching context for project:", project.name); + + // Create a deep copy to avoid reference issues + const contextCacheCopy = JSON.parse(JSON.stringify(contextCache)); + // Store in memory cache - this.memoryCache.set(cacheKey, contextCache); + this.memoryCache.set(cacheKey, contextCacheCopy); + // Store in file cache - await this.vault.adapter.write(cachePath, JSON.stringify(contextCache)); + await this.vault.adapter.write(cachePath, JSON.stringify(contextCacheCopy)); } catch (error) { logError("Error writing to project context cache:", error); + throw error; // Re-throw to maintain error propagation } } @@ -209,7 +279,7 @@ export class ProjectContextCache { youtubeContexts: {}, fileContexts: {}, timestamp: Date.now(), - markdownNeedsReload: false, + markdownNeedsReload: true, }; } @@ -314,6 +384,10 @@ export class ProjectContextCache { `[clearForProject] Project ${project.name}: Main project cache file not found (already deleted or never existed): ${cachePath}` ); } + // Clean up the mutex for this project to prevent memory leaks + this.projectMutexMap.delete(project.id); + logInfo(`[clearForProject] Cleaned up mutex for project: ${project.name}`); + logInfo(`[clearForProject] Completed for project: ${project.name}`); } catch (error) { logError(`[clearForProject] Error for project ${project.name} (ID: ${project.id}):`, error); @@ -332,47 +406,49 @@ export class ProjectContextCache { project: ProjectConfig, forceReloadAllRemotes: boolean = false ): Promise { - const cache = await this.get(project); - if (cache) { - cache.markdownContext = ""; - cache.markdownNeedsReload = true; + await this.updateCacheSafely( + project, + (cache) => { + cache.markdownContext = ""; + cache.markdownNeedsReload = true; - if (forceReloadAllRemotes) { - cache.webContexts = {}; - cache.youtubeContexts = {}; - logInfo(`Flagged Web/YouTube contexts for full reload for project ${project.name}`); - } + if (forceReloadAllRemotes) { + cache.webContexts = {}; + cache.youtubeContexts = {}; + logInfo(`Flagged Web/YouTube contexts for full reload for project ${project.name}`); + } - await this.set(project, cache); + // Also clean up any file references that no longer match the project's patterns + const cleanedCache = this.cleanupFileReferencesInCache(project, cache); - // Also clean up any file references that no longer match the project's patterns - await this.cleanupProjectFileReferences(project); - - logInfo(`Invalidated markdown context for project ${project.name}`); - } + logInfo(`Invalidated markdown context for project ${project.name}`); + return cleanedCache; + }, + true + ); } /** * Update the markdown context for a project */ async updateMarkdownContext(project: ProjectConfig, content: string): Promise { - const cache = (await this.get(project)) || this.createEmptyCache(); - cache.markdownContext = content; - cache.markdownNeedsReload = false; - await this.set(project, cache); - logInfo(`Updated markdown context for project ${project.name}`); + return await this.updateCacheSafely(project, (cache) => { + cache.markdownContext = content; + cache.markdownNeedsReload = false; + logInfo(`Updated markdown context for project ${project.name}`); + return cache; + }); } /** * Clear only the markdown context for a project */ async clearMarkdownContext(project: ProjectConfig): Promise { - const cache = await this.get(project); - if (cache) { + await this.updateCacheSafely(project, (cache) => { cache.markdownContext = ""; cache.markdownNeedsReload = true; - await this.set(project, cache); - } + return cache; + }); } //=========================================================================== @@ -380,10 +456,41 @@ export class ProjectContextCache { // Non-markdown files cached in fileCache after processing by FileParserManager //=========================================================================== + /** + * Get file content from project cache or reuse from universal cache if available + * This method efficiently searches across all available caches to find file content + */ + async getOrReuseFileContext(project: ProjectConfig, filePath: string): Promise { + try { + // 1. Try to get from project cache first + const projectContent = await this.getFileContext(project, filePath); + if (projectContent) { + return projectContent; + } + + // 2. Search other projects as fallback + const result = await this.searchOtherProjectsForFile(filePath); + if (result) { + // Associate with current project + await this.associateCacheWithProject(project, filePath, result.cacheKey); + logInfo( + `Reused cached content from other project for: ${filePath} in project ${project.name}` + ); + return result.content; + } + + // No content found in any cache + return null; + } catch (error) { + logError(`Error in getOrReuseFileContext for ${filePath} in project ${project.name}:`, error); + return null; + } + } + /** * Get content for a specific file in a project */ - async getFileContext(project: ProjectConfig, filePath: string): Promise { + protected async getFileContext(project: ProjectConfig, filePath: string): Promise { try { // Ensure filePath is valid before proceeding if (!filePath || typeof filePath !== "string") { @@ -445,8 +552,7 @@ export class ProjectContextCache { * Add or update a file in a project's context */ async setFileContext(project: ProjectConfig, filePath: string, content: string): Promise { - try { - const cache = (await this.get(project)) || this.createEmptyCache(); + return await this.updateCacheSafelyAsync(project, async (cache) => { if (!cache.fileContexts) { cache.fileContexts = {}; } @@ -469,60 +575,122 @@ export class ProjectContextCache { cacheKey, }; - await this.set(project, cache); logInfo(`Added/updated file context for ${filePath} in project ${project.name}`); - } catch (error) { - logError(`Error setting file context for ${filePath}:`, error); - } + return cache; + }); } /** * Remove a file from a project's context */ async removeFileContext(project: ProjectConfig, filePath: string): Promise { - try { - const cache = await this.get(project); - if (cache && cache.fileContexts[filePath]) { + return await this.updateCacheSafelyAsync(project, async (cache) => { + if (cache.fileContexts && cache.fileContexts[filePath]) { // Get the cache key before removing from project cache const { cacheKey } = cache.fileContexts[filePath]; // Remove from project cache delete cache.fileContexts[filePath]; - await this.set(project, cache); // Remove from file cache await this.fileCache.remove(cacheKey); logInfo(`Removed file context for ${filePath} in project ${project.name}`); } + return cache; + }); + } + + /** + * Search all existing projects for cached content of a file + * If found, migrate that content to universal cache and return it + */ + private async searchOtherProjectsForFile( + filePath: string + ): Promise<{ cacheKey: string; content: string } | null> { + try { + const settings = getSettings(); + const projects = settings.projectList || []; + + if (projects.length === 0) { + return null; + } + + logInfo(`Searching other projects for file: ${filePath}`); + + for (const project of projects) { + // Skip projects without cache + const cache = await this.get(project); + if (!cache || !cache.fileContexts) { + continue; + } + + // Check if this project has the file cached + if (cache.fileContexts[filePath]) { + const { cacheKey } = cache.fileContexts[filePath]; + if (!cacheKey) continue; + + // Try to get content from this project's cache + const content = await this.fileCache.get(cacheKey); + if (content) { + logInfo(`Found content for file ${filePath} in project ${project.name}`); + return { content, cacheKey }; + } + } + } + + logInfo(`No content found in any project for file: ${filePath}`); + return null; } catch (error) { - logError(`Error removing file context for ${filePath}:`, error); + logError(`Error searching other projects for file ${filePath}:`, error); + return null; } } /** - * Update project file references based on inclusion/exclusion patterns. - * Removes references to files that no longer match patterns, but keeps their content cached. + * Associate an existing cache with a specific project, creates the project reference to that cache */ - async cleanupProjectFileReferences(project: ProjectConfig): Promise { - try { - const cache = await this.get(project); - if (!cache || !cache.fileContexts) { - return; + async associateCacheWithProject( + project: ProjectConfig, + filePath: string, + cacheKey: string + ): Promise { + return await this.updateCacheSafelyAsync(project, async (cache) => { + if (!cache.fileContexts) { + cache.fileContexts = {}; } - const { inclusions, exclusions } = getMatchingPatterns({ - inclusions: project.contextSource.inclusions, - exclusions: project.contextSource.exclusions, - isProject: true, - }); + // Update project context to reference the other cache key directly + cache.fileContexts[filePath] = { + timestamp: Date.now(), + cacheKey: cacheKey, + }; - let removedCount = 0; - const updatedFileContexts: typeof cache.fileContexts = {}; + logInfo(`Associated cache with project ${project.name} for file: ${filePath}`); + return cache; + }); + } - // Check each file against the patterns - for (const filePath in cache.fileContexts) { - const file = this.vault.getAbstractFileByPath(filePath); + /** + * Helper method to perform file reference cleanup logic on a cache object + */ + private cleanupFileReferencesInCache(project: ProjectConfig, cache: ContextCache): ContextCache { + if (!cache.fileContexts) { + return cache; + } + + const { inclusions, exclusions } = getMatchingPatterns({ + inclusions: project.contextSource.inclusions, + exclusions: project.contextSource.exclusions, + isProject: true, + }); + + let removedCount = 0; + const updatedFileContexts: typeof cache.fileContexts = {}; + + // Check each file against the patterns + for (const filePath in cache.fileContexts) { + const file = this.vault.getAbstractFileByPath(filePath); // If file no longer exists or doesn't match patterns, remove its reference if (!(file instanceof TFile) || !shouldIndexFile(file, inclusions, exclusions, true)) { @@ -534,14 +702,29 @@ export class ProjectContextCache { } } - // Only update if we actually removed something - if (removedCount > 0) { - cache.fileContexts = updatedFileContexts; - await this.set(project, cache); - logInfo( - `Removed ${removedCount} file references from project ${project.name} that no longer match inclusion patterns` - ); - } + // Only update if we actually removed something + if (removedCount > 0) { + cache.fileContexts = updatedFileContexts; + logInfo( + `Removed ${removedCount} file references from project ${project.name} that no longer match inclusion patterns` + ); + } + + return cache; + } + + /** + * Update project file references based on inclusion/exclusion patterns. + * Removes references to files that no longer match patterns, but keeps their content cached. + */ + async cleanupProjectFileReferences(project: ProjectConfig): Promise { + logInfo(`[cleanupProjectFileReferences] Starting for project: ${project.name}`); + try { + await this.updateCacheSafely( + project, + (cache) => this.cleanupFileReferencesInCache(project, cache), + true + ); } catch (error) { logError(`Error cleaning up project file references for ${project.name}:`, error); } @@ -598,33 +781,82 @@ export class ProjectContextCache { return contextCacheToUpdate; } + updateProjectMarkdownFilesFromPatterns( + project: ProjectConfig, + contextCacheToUpdate: ContextCache, + projectAllFiles: TFile[] + ): ContextCache { + try { + if (!contextCacheToUpdate.fileContexts) { + contextCacheToUpdate.fileContexts = {}; + } + + const allFiles = projectAllFiles.filter((file) => file.extension === "md"); + let addedCount = 0; + + for (const file of allFiles) { + if (contextCacheToUpdate.fileContexts[file.path]) { + continue; + } + const cacheKey = this.fileCache.getCacheKey(file, project.id); + contextCacheToUpdate.fileContexts[file.path] = { + timestamp: Date.now(), + cacheKey, + }; + addedCount++; + } + + if (addedCount > 0) { + logInfo( + `[updateProjectFilesFromPatterns] Project ${project.name}: Added ${addedCount} new file references to context (in memory).` + ); + } + + logInfo( + `[updateProjectFilesFromPatterns] Completed for project: ${project.name}. Total markdown fileContexts in memory: ${Object.keys(contextCacheToUpdate.fileContexts).length}` + ); + } catch (error) { + logError(`[updateProjectFilesFromPatterns] Error for project ${project.name}:`, error); + } + return contextCacheToUpdate; + } + //=========================================================================== // WEB CONTEXT OPERATIONS //=========================================================================== /** - * Remove a web URL from a project's context + * Remove a web URLs from a project's context */ - async removeWebUrl(project: ProjectConfig, url: string): Promise { - const cache = await this.get(project); - if (cache?.webContexts?.[url]) { - delete cache.webContexts[url]; - await this.set(project, cache); - logInfo(`Removed web context for URL ${url} in project ${project.name}`); - } + + async removeWebUrls(project: ProjectConfig, urls: string[]): Promise { + if (!urls.length) return; + + await this.updateCacheSafely(project, (cache) => { + if (cache.webContexts) { + for (const url of urls) { + if (cache.webContexts[url]) { + delete cache.webContexts[url]; + } + } + logInfo(`Removed web contexts for URLs ${urls.join(", ")} in project ${project.name}`); + } + return cache; + }); } /** * Add or update a web URL in a project's context */ async updateWebUrl(project: ProjectConfig, url: string, content: string): Promise { - const cache = (await this.get(project)) || this.createEmptyCache(); - if (!cache.webContexts) { - cache.webContexts = {}; - } - cache.webContexts[url] = content; - await this.set(project, cache); - logInfo(`Updated web context for URL ${url} in project ${project.name}`); + return await this.updateCacheSafely(project, (cache) => { + if (!cache.webContexts) { + cache.webContexts = {}; + } + cache.webContexts[url] = content; + logInfo(`Updated web context for URL ${url} in project ${project.name}`); + return cache; + }); } //=========================================================================== @@ -632,27 +864,123 @@ export class ProjectContextCache { //=========================================================================== /** - * Remove a YouTube URL from a project's context + * Remove a YouTube URLs from a project's context */ - async removeYoutubeUrl(project: ProjectConfig, url: string): Promise { - const cache = await this.get(project); - if (cache?.youtubeContexts?.[url]) { - delete cache.youtubeContexts[url]; - await this.set(project, cache); - logInfo(`Removed YouTube context for URL ${url} in project ${project.name}`); - } + + async removeYoutubeUrls(project: ProjectConfig, urls: string[]): Promise { + if (!urls.length) return; + + await this.updateCacheSafely(project, (cache) => { + if (cache.youtubeContexts) { + for (const url of urls) { + if (cache.youtubeContexts[url]) { + delete cache.youtubeContexts[url]; + } + } + logInfo( + `removeYoutubeUrls: Removed YouTube contexts for URLs ${urls.join(", ")} in project ${project.name}` + ); + } + return cache; + }); } /** * Add or update a YouTube URL in a project's context */ async updateYoutubeUrl(project: ProjectConfig, url: string, content: string): Promise { - const cache = (await this.get(project)) || this.createEmptyCache(); - if (!cache.youtubeContexts) { - cache.youtubeContexts = {}; - } - cache.youtubeContexts[url] = content; - await this.set(project, cache); - logInfo(`Updated YouTube context for URL ${url} in project ${project.name}`); + return await this.updateCacheSafely(project, (cache) => { + if (!cache.youtubeContexts) { + cache.youtubeContexts = {}; + } + cache.youtubeContexts[url] = content; + logInfo(`Updated YouTube context for URL ${url} in project ${project.name}`); + return cache; + }); + } + + //=========================================================================== + // EXTERNAL SAFE OPERATIONS + //=========================================================================== + + /** + * Safe external method for bulk cache updates + * This is for external modules that need to perform complex updates safely + * @param project + * @param updateFn + * @param skipIfEmpty - If true, skip the update when cache is empty instead of throwing an error + */ + async updateCacheSafely( + project: ProjectConfig, + updateFn: (cache: ContextCache) => ContextCache, + skipIfEmpty: boolean = false + ): Promise { + const mutex = await this.getOrCreateProjectMutex(project); + + return await mutex.runExclusive(async () => { + try { + const cache = await this.get(project); + if (!cache) { + if (skipIfEmpty) { + return; + } + throw new Error( + `Project: ${project.name} context cache not found, please invoke getOrInitializeCache method before invoke update context cache` + ); + } + const updatedCache = updateFn(cache); + await this.setWithoutMutex(project, updatedCache); + } catch (error) { + logError(`Error updating cache for project ${project.name}:`, error); + throw error; + } + }); + } + + /** + * Safe external method for async cache updates + * This is for external modules that need to perform async updates safely + * @param project + * @param updateFn + * @param skipIfEmpty - If true, skip the update when cache is empty instead of throwing an error + */ + async updateCacheSafelyAsync( + project: ProjectConfig, + updateFn: (cache: ContextCache) => Promise, + skipIfEmpty: boolean = false + ): Promise { + const mutex = await this.getOrCreateProjectMutex(project); + + return await mutex.runExclusive(async () => { + try { + const cache = await this.get(project); + if (!cache) { + if (skipIfEmpty) { + return; + } + throw new Error( + `Project: ${project.name} context cache not found, please invoke getOrInitializeCache method before invoke update context cache` + ); + } + const updatedCache = await updateFn(cache); + await this.setWithoutMutex(project, updatedCache); + } catch (error) { + logError(`Error updating cache for project ${project.name}:`, error); + throw error; + } + }); + } + + /** + * Safe external method for setting complete cache + * Use this instead of direct set() calls from external modules + */ + async setCacheSafely(project: ProjectConfig, contextCache: ContextCache): Promise { + const mutex = await this.getOrCreateProjectMutex(project); + + return await mutex.runExclusive(async () => { + logInfo(`External safe set for project: ${project.name}`); + return await this.setWithoutMutex(project, contextCache); + }); } } diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index 76f94e4a..60ebf4f8 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -8,6 +8,7 @@ import { useModelKey, useSelectedTextContexts, } from "@/aiParams"; +import { useProjectContextStatus } from "@/hooks/useProjectContextStatus"; import { ChainType } from "@/chainFactory"; import { ChatControls, reloadCurrentProject } from "@/components/chat-components/ChatControls"; @@ -30,6 +31,7 @@ import { err2String } from "@/utils"; import { Buffer } from "buffer"; import { Notice, TFile } from "obsidian"; import React, { useCallback, useContext, useEffect, useRef, useState } from "react"; +import ProgressCard from "@/components/project/progress-card"; type ChatMode = "default" | "project"; @@ -66,7 +68,29 @@ const Chat: React.FC = ({ const [includeActiveNote, setIncludeActiveNote] = useState(false); const [selectedImages, setSelectedImages] = useState([]); const [showChatUI, setShowChatUI] = useState(false); + // null: keep default behavior; true: show; false: hide + const [progressCardVisible, setProgressCardVisible] = useState(null); + const [selectedTextContexts] = useSelectedTextContexts(); + const projectContextStatus = useProjectContextStatus(); + + // Calculate whether to show ProgressCard based on status and user preference + const shouldShowProgressCard = () => { + if (selectedChain !== ChainType.PROJECT_CHAIN) return false; + + // If user has explicitly set visibility, respect that choice + if (progressCardVisible !== null) { + return progressCardVisible; + } + + // Default behavior: show for loading/error, hide for success + return projectContextStatus === "loading" || projectContextStatus === "error"; + }; + + // Reset user preference when status changes to allow default behavior + useEffect(() => { + setProgressCardVisible(null); + }, [projectContextStatus]); const [previousMode, setPreviousMode] = useState(null); const [selectedChain, setSelectedChain] = useChainType(); @@ -535,38 +559,54 @@ const Chat: React.FC = ({ onReplaceChat={setInputMessage} showHelperComponents={selectedChain !== ChainType.PROJECT_CHAIN} /> - handleSaveAsNote()} - onLoadHistory={handleLoadHistory} - onModeChange={(newMode) => { - setPreviousMode(selectedChain); - // Hide chat UI when switching to project mode - if (newMode === ChainType.PROJECT_CHAIN) { - setShowChatUI(false); - } - }} - /> - handleStopGenerating(ABORT_REASON.USER_STOPPED)} - app={app} - contextNotes={contextNotes} - setContextNotes={setContextNotes} - includeActiveNote={includeActiveNote} - setIncludeActiveNote={setIncludeActiveNote} - mention={mention} - selectedImages={selectedImages} - onAddImage={(files: File[]) => setSelectedImages((prev) => [...prev, ...files])} - setSelectedImages={setSelectedImages} - disableModelSwitch={selectedChain === ChainType.PROJECT_CHAIN} - selectedTextContexts={selectedTextContexts} - onRemoveSelectedText={handleRemoveSelectedText} - /> + {shouldShowProgressCard() ? ( +
+ { + setProgressCardVisible(false); + }} + /> +
+ ) : ( + <> + handleSaveAsNote()} + onLoadHistory={handleLoadHistory} + onModeChange={(newMode) => { + setPreviousMode(selectedChain); + // Hide chat UI when switching to project mode + if (newMode === ChainType.PROJECT_CHAIN) { + setShowChatUI(false); + } + }} + /> + handleStopGenerating(ABORT_REASON.USER_STOPPED)} + app={app} + contextNotes={contextNotes} + setContextNotes={setContextNotes} + includeActiveNote={includeActiveNote} + setIncludeActiveNote={setIncludeActiveNote} + mention={mention} + selectedImages={selectedImages} + onAddImage={(files: File[]) => setSelectedImages((prev) => [...prev, ...files])} + setSelectedImages={setSelectedImages} + disableModelSwitch={selectedChain === ChainType.PROJECT_CHAIN} + selectedTextContexts={selectedTextContexts} + onRemoveSelectedText={handleRemoveSelectedText} + showProgressCard={() => { + setProgressCardVisible(true); + }} + /> + + )} ); @@ -597,6 +637,9 @@ const Chat: React.FC = ({ } }} showChatUI={(v) => setShowChatUI(v)} + onProjectClose={() => { + setProgressCardVisible(null); + }} /> )} diff --git a/src/components/chat-components/ChatContextMenu.tsx b/src/components/chat-components/ChatContextMenu.tsx index c483ef5d..da76d30a 100644 --- a/src/components/chat-components/ChatContextMenu.tsx +++ b/src/components/chat-components/ChatContextMenu.tsx @@ -1,9 +1,13 @@ -import { Plus, X } from "lucide-react"; +import { AlertCircle, CheckCircle, CircleDashed, Loader2, Plus, X } from "lucide-react"; import { TFile } from "obsidian"; import React from "react"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; import { SelectedTextContext } from "@/types/message"; +import { ChainType } from "@/chainFactory"; +import { Separator } from "@/components/ui/separator"; +import { useChainType } from "@/aiParams"; +import { useProjectContextStatus } from "@/hooks/useProjectContextStatus"; interface ChatContextMenuProps { activeNote: TFile | null; @@ -14,6 +18,7 @@ interface ChatContextMenuProps { onRemoveContext: (path: string) => void; onRemoveUrl: (url: string) => void; onRemoveSelectedText?: (id: string) => void; + showProgressCard: () => void; } function ContextNote({ @@ -102,7 +107,11 @@ export const ChatContextMenu: React.FC = ({ onRemoveContext, onRemoveUrl, onRemoveSelectedText, + showProgressCard, }) => { + const [currentChain] = useChainType(); + const contextStatus = useProjectContextStatus(); + const uniqueNotes = React.useMemo(() => { const notesMap = new Map(contextNotes.map((note) => [note.path, note])); @@ -125,6 +134,20 @@ export const ChatContextMenu: React.FC = ({ selectedTextContexts.length > 0 || !!activeNote; + // Get contextStatus from the shared hook + const getContextStatusIcon = () => { + switch (contextStatus) { + case "success": + return ; + case "loading": + return ; + case "error": + return ; + case "initial": + return ; + } + }; + return (
@@ -166,6 +189,22 @@ export const ChatContextMenu: React.FC = ({ /> ))}
+ + {currentChain === ChainType.PROJECT_CHAIN && ( + <> + +
+ +
+ + )}
); }; diff --git a/src/components/chat-components/ChatInput.tsx b/src/components/chat-components/ChatInput.tsx index 0c8d9a25..165e011a 100644 --- a/src/components/chat-components/ChatInput.tsx +++ b/src/components/chat-components/ChatInput.tsx @@ -1,6 +1,6 @@ import { - ProjectConfig, getCurrentProject, + ProjectConfig, subscribeToProjectChange, useChainType, useModelKey, @@ -71,6 +71,7 @@ interface ChatInputProps { disableModelSwitch?: boolean; selectedTextContexts?: SelectedTextContext[]; onRemoveSelectedText?: (id: string) => void; + showProgressCard: () => void; } const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( @@ -93,6 +94,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( disableModelSwitch, selectedTextContexts, onRemoveSelectedText, + showProgressCard, }, ref ) => { @@ -509,6 +511,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>( onRemoveUrl={(url: string) => setContextUrls((prev) => prev.filter((u) => u !== url))} selectedTextContexts={selectedTextContexts} onRemoveSelectedText={onRemoveSelectedText} + showProgressCard={showProgressCard} /> {selectedImages.length > 0 && ( diff --git a/src/components/chat-components/ContextControl.tsx b/src/components/chat-components/ContextControl.tsx index ed6a3f57..795610ca 100644 --- a/src/components/chat-components/ContextControl.tsx +++ b/src/components/chat-components/ContextControl.tsx @@ -20,6 +20,7 @@ interface ChatControlsProps { onRemoveUrl: (url: string) => void; selectedTextContexts?: SelectedTextContext[]; onRemoveSelectedText?: (id: string) => void; + showProgressCard: () => void; } const ContextControl: React.FC = ({ @@ -34,6 +35,7 @@ const ContextControl: React.FC = ({ onRemoveUrl, selectedTextContexts, onRemoveSelectedText, + showProgressCard, }) => { const [selectedChain] = useChainType(); @@ -86,6 +88,7 @@ const ContextControl: React.FC = ({ onRemoveUrl={onRemoveUrl} selectedTextContexts={selectedTextContexts} onRemoveSelectedText={onRemoveSelectedText} + showProgressCard={showProgressCard} /> ); }; diff --git a/src/components/chat-components/ProjectList.tsx b/src/components/chat-components/ProjectList.tsx index 8f4d982f..6389c96d 100644 --- a/src/components/chat-components/ProjectList.tsx +++ b/src/components/chat-components/ProjectList.tsx @@ -131,6 +131,7 @@ export const ProjectList = memo( showChatUI, onClose, inputRef, + onProjectClose, }: { className?: string; projects: ProjectConfig[]; @@ -142,6 +143,7 @@ export const ProjectList = memo( showChatUI: (v: boolean) => void; onClose: () => void; inputRef: React.RefObject; + onProjectClose: () => void; }): React.ReactElement => { const [isOpen, setIsOpen] = useState(defaultOpen); const [showChatInput, setShowChatInput] = useState(false); @@ -285,6 +287,7 @@ export const ProjectList = memo( size="icon" onClick={() => { enableOrDisableProject(false); + onProjectClose(); }} aria-label="Close Current Project" > diff --git a/src/components/project/progress-card.tsx b/src/components/project/progress-card.tsx new file mode 100644 index 00000000..5fea7806 --- /dev/null +++ b/src/components/project/progress-card.tsx @@ -0,0 +1,218 @@ +import * as React from "react"; +import { useState } from "react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { Badge } from "@/components/ui/badge"; +import { + AlertCircle, + ChevronDown, + ChevronRight, + FileText, + Loader2, + RotateCcw, + X, +} from "lucide-react"; +import { FailedItem, useProjectContextLoad } from "@/aiParams"; +import { Button } from "@/components/ui/button"; +import { TruncatedText } from "@/components/TruncatedText"; +import CopilotPlugin from "@/main"; +import { logError } from "@/logger"; + +interface ProgressCardProps { + plugin?: CopilotPlugin; + setHiddenCard: (hidden: boolean) => void; +} + +export default function ProgressCard({ plugin, setHiddenCard }: ProgressCardProps) { + const [contextLoadState] = useProjectContextLoad(); + const totalFiles = contextLoadState.total; + const successFiles = contextLoadState.success; + const failedFiles = contextLoadState.failed; + const processingFiles = contextLoadState.processingFiles; + + // Control file list expand/collapse state + const [isProcessingExpanded, setIsProcessingExpanded] = useState(false); + const [isFailedExpanded, setIsFailedExpanded] = useState(false); + + const processedFilesLen = successFiles.length + failedFiles.length; + const progressPercentage = + totalFiles.length > 0 ? Math.round((processedFilesLen / totalFiles.length) * 100) : 0; + + const getFailedItemDisplayName = (item: FailedItem): string => { + return item.path; + }; + + // TODO(emt-lin): maybe use it in the future + /*const handleRetryAllFailed = () => { + console.log("Retrying all failed items"); + };*/ + + const handleRetryFailedItem = async (item: FailedItem) => { + if (!plugin?.projectManager) { + logError("ProjectManager not available"); + return; + } + + try { + await plugin.projectManager.retryFailedItem(item); + } catch (error) { + logError(`Error retrying failed item: ${error}`); + } + }; + + return ( + + + +
+ + Context Loading +
+ + + + + {/* Total progress display */} +
+
+
+ Total progress + + (Success:{" "} + {successFiles.length}, + Failed: {failedFiles.length}) + +
+ + {processedFilesLen}/{totalFiles.length} ({progressPercentage}%) + +
+ +
+ + {/* Currently processing file */} + {processingFiles.length > 0 && ( +
+
setIsProcessingExpanded(!isProcessingExpanded)} + > + + Processing + {isProcessingExpanded ? ( + + ) : ( + + )} +
+ + {isProcessingExpanded && ( +
+ {processingFiles.map((fileName, index) => ( +
+
+ + {fileName} + +
+ ))} +
+ )} +
+ )} + + {/* Failed to process the file */} + {failedFiles.length > 0 && ( +
+
+
setIsFailedExpanded(!isFailedExpanded)} + > + + Failed + + {failedFiles.length} files + + {isFailedExpanded ? ( + + ) : ( + + )} +
+ {/*todo(emt-lin): in the future, we can add all failed files to retry*/} + {/**/} +
+ + {isFailedExpanded && ( +
+ {failedFiles.map((failedItem: FailedItem, index: number) => ( +
+
+
+
+ + {getFailedItemDisplayName(failedItem)} + +
+
+
+ {failedItem.error && ( + + Loading Error: + {failedItem.error} + + )} +
+
+ +
+ ))} +
+ )} +
+ )} + + + ); +} diff --git a/src/components/ui/progress.tsx b/src/components/ui/progress.tsx new file mode 100644 index 00000000..9f6449c8 --- /dev/null +++ b/src/components/ui/progress.tsx @@ -0,0 +1,26 @@ +import * as React from "react"; +import * as ProgressPrimitive from "@radix-ui/react-progress"; + +import { cn } from "@/lib/utils"; + +const Progress = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, value, ...props }, ref) => ( + + + +)); +Progress.displayName = ProgressPrimitive.Root.displayName; + +export { Progress }; diff --git a/src/hooks/useProjectContextStatus.ts b/src/hooks/useProjectContextStatus.ts new file mode 100644 index 00000000..11e54b02 --- /dev/null +++ b/src/hooks/useProjectContextStatus.ts @@ -0,0 +1,50 @@ +import { ChainType } from "@/chainFactory"; +import { useChainType, useProjectLoading, useProjectContextLoad } from "@/aiParams"; + +/** + * Hook to calculate the project context status based on project loading state and context load state. + * Returns one of: 'initial', 'loading', 'success', 'error' + * + * Status meanings: + * - 'initial': Project context loading has not started yet or not in project mode + * - 'loading': Project context is currently being loaded or files are being processed + * - 'success': All files have been processed successfully without any errors + * - 'error': One or more files have failed to process during the loading operation + */ + +export type ProjectContextStatus = "initial" | "loading" | "success" | "error"; + +export function useProjectContextStatus(): ProjectContextStatus { + const [currentChain] = useChainType(); + const [isProjectLoading] = useProjectLoading(); + const [contextLoadState] = useProjectContextLoad(); + + const contextStatus = (() => { + // Only calculate status for project mode + if (currentChain !== ChainType.PROJECT_CHAIN) { + return "initial"; + } + + const { total, success, failed, processingFiles } = contextLoadState; + + // Loading state: when project is loading or files are being processed + if (isProjectLoading || processingFiles.length > 0) { + return "loading"; + } + + // Error state: when there are failed files + if (failed.length > 0) { + return "error"; + } + + // Success state: when all files have been processed successfully + if (total.length > 0 && success.length === total.length) { + return "success"; + } + + // Initial state: when no context loading has started yet + return "initial"; + })(); + + return contextStatus; +} diff --git a/src/mentions/Mention.ts b/src/mentions/Mention.ts index 82eaab14..fbb3b45a 100644 --- a/src/mentions/Mention.ts +++ b/src/mentions/Mention.ts @@ -1,11 +1,13 @@ import { ImageProcessor } from "@/imageProcessing/imageProcessor"; import { BrevilabsClient, Url4llmResponse } from "@/LLMProviders/brevilabsClient"; -import { isYoutubeUrl } from "@/utils"; +import { err2String, isYoutubeUrl } from "@/utils"; +import { logError } from "@/logger"; export interface MentionData { type: string; original: string; processed?: string; + error?: string; } export class Mention { @@ -41,24 +43,30 @@ export class Mention { .filter((url) => !isYoutubeUrl(url)); } - async processUrl(url: string): Promise { + async processUrl(url: string): Promise { try { return await this.brevilabsClient.url4llm(url); } catch (error) { - console.error(`Error processing URL ${url}:`, error); - return { response: url, elapsed_time_ms: 0 }; + const msg = err2String(error); + logError(`Error processing URL ${url}: ${msg}`); + return { response: url, elapsed_time_ms: 0, error: msg }; } } // For non-youtube URLs - async processUrls(text: string): Promise<{ urlContext: string; imageUrls: string[] }> { + async processUrls(text: string): Promise<{ + urlContext: string; + imageUrls: string[]; + processedErrorUrls: Record; + }> { const urls = this.extractUrls(text); let urlContext = ""; const imageUrls: string[] = []; + const processedErrorUrls: Record = {}; // Return empty string if no URLs to process if (urls.length === 0) { - return { urlContext: "", imageUrls: [] }; + return { urlContext, imageUrls, processedErrorUrls }; } // Process all URLs concurrently @@ -75,6 +83,7 @@ export class Mention { type: "url", original: url, processed: processed.response, + error: processed.error, }); } return this.mentions.get(url); @@ -87,9 +96,13 @@ export class Mention { if (urlData?.processed) { urlContext += `\n\n\n${urlData.original}\n\n${urlData.processed}\n\n`; } + + if (urlData?.error) { + processedErrorUrls[urlData.original] = urlData.error; + } }); - return { urlContext, imageUrls }; + return { urlContext, imageUrls, processedErrorUrls }; } getMentions(): Map { diff --git a/src/search/searchUtils.ts b/src/search/searchUtils.ts index 9a890a2d..db732fdb 100644 --- a/src/search/searchUtils.ts +++ b/src/search/searchUtils.ts @@ -147,7 +147,7 @@ export function shouldIndexFile( return false; } - // Project：Only the included files need to be processed. + // Project: Only the included files need to be processed. if (isProject && !inclusions) { return false; } diff --git a/src/styles/tailwind.css b/src/styles/tailwind.css index e4615e0a..7e960ba8 100644 --- a/src/styles/tailwind.css +++ b/src/styles/tailwind.css @@ -1,3 +1,7 @@ +/* + Copilot Plugin Css + This file is generated by Copilot plugin. +*/ @import "tailwindcss/base"; @import "tailwindcss/components"; @import "tailwindcss/utilities"; diff --git a/src/tests/projectContextCache.test.ts b/src/tests/projectContextCache.test.ts index b80b4162..58f06550 100644 --- a/src/tests/projectContextCache.test.ts +++ b/src/tests/projectContextCache.test.ts @@ -1,8 +1,25 @@ import { ProjectConfig } from "@/aiParams"; -import { ProjectContextCache } from "@/cache/projectContextCache"; +import { ContextCache, ProjectContextCache } from "@/cache/projectContextCache"; // Mock dependencies -jest.mock("obsidian"); +jest.mock("obsidian", () => ({ + TFile: class MockTFile { + path: string; + extension: string; + basename: string; + stat: any; + + constructor(path: string, extension: string, basename: string, stat: any) { + this.path = path; + this.extension = extension; + this.basename = basename; + this.stat = stat; + } + }, + TAbstractFile: class MockTAbstractFile {}, + Vault: class MockVault {}, +})); + jest.mock("@/logger", () => ({ logInfo: jest.fn(), logError: jest.fn(), @@ -94,19 +111,16 @@ describe("ProjectContextCache", () => { let mockProject: ProjectConfig; // Mock files - const mockMarkdownFile = { - path: "test/file.md", - extension: "md", - basename: "file", - stat: { mtime: Date.now(), size: 100 }, - } as any; + const { TFile: MockedTFile } = jest.requireMock("obsidian"); + const mockMarkdownFile = new MockedTFile("test/file.md", "md", "file", { + mtime: Date.now(), + size: 100, + }); - const mockPdfFile = { - path: "test/document.pdf", - extension: "pdf", - basename: "document", - stat: { mtime: Date.now(), size: 200 }, - } as any; + const mockPdfFile = new MockedTFile("test/document.pdf", "pdf", "document", { + mtime: Date.now(), + size: 200, + }); beforeEach(() => { // Reset mocks @@ -123,7 +137,12 @@ describe("ProjectContextCache", () => { return null; }); - // Get actual instance + // Reset vault adapter mocks to default behavior + mockApp.vault.adapter.exists.mockResolvedValue(false); + mockApp.vault.adapter.read.mockResolvedValue("{}"); + mockApp.vault.adapter.write.mockResolvedValue(undefined); + + // Get actual instance and clear any existing cache projectContextCache = ProjectContextCache.getInstance(); // Mock project with minimal properties for testing @@ -141,11 +160,14 @@ describe("ProjectContextCache", () => { const filePath = "test/document.pdf"; const content = "PDF content"; + // Initialize a default cache for testing + await projectContextCache.getOrInitializeCache(mockProject); + // Store content await projectContextCache.setFileContext(mockProject, filePath, content); // Get content - const retrievedContent = await projectContextCache.getFileContext(mockProject, filePath); + const retrievedContent = await projectContextCache.getOrReuseFileContext(mockProject, filePath); // Verify the content was retrieved (note: implementation may return an object instead of string) expect(retrievedContent).toBeDefined(); @@ -176,6 +198,9 @@ describe("ProjectContextCache", () => { }); test("should clean up project file references", async () => { + // Initialize a default cache for testing + await projectContextCache.getOrInitializeCache(mockProject); + // First add some context await projectContextCache.setFileContext(mockProject, mockPdfFile.path, "PDF content"); @@ -187,4 +212,440 @@ describe("ProjectContextCache", () => { const projectCache = await projectContextCache.get(mockProject); expect(projectCache).toBeDefined(); }); + + test("should update project markdown files from patterns", () => { + // Create an empty context cache to update + const contextCache = { + markdownContext: "", + markdownNeedsReload: true, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Create a test file list containing Markdown and non-Markdown files + const testFiles = [ + { + path: "test/file1.md", + extension: "md", + basename: "file1", + stat: { mtime: Date.now(), size: 100 }, + }, + { + path: "test/file2.md", + extension: "md", + basename: "file2", + stat: { mtime: Date.now(), size: 200 }, + }, + { + path: "test/document.pdf", + extension: "pdf", + basename: "document", + stat: { mtime: Date.now(), size: 300 }, + }, + ]; + + // Call the method, passing only Markdown files + const updatedCache = projectContextCache.updateProjectMarkdownFilesFromPatterns( + mockProject, + contextCache, + testFiles as any + ); + + // Verify that only Markdown files were added to the cache + expect(Object.keys(updatedCache.fileContexts).length).toBe(2); + expect(updatedCache.fileContexts["test/file1.md"]).toBeDefined(); + expect(updatedCache.fileContexts["test/file2.md"]).toBeDefined(); + expect(updatedCache.fileContexts["test/document.pdf"]).toBeUndefined(); + + // Verify that each file entry contains the necessary properties + Object.values(updatedCache.fileContexts).forEach((entry) => { + expect(entry).toHaveProperty("timestamp"); + expect(entry).toHaveProperty("cacheKey"); + expect(typeof entry.timestamp).toBe("number"); + expect(typeof entry.cacheKey).toBe("string"); + }); + }); + + test("should update and remove web URLs", async () => { + // Create initial context cache + const initialCache = { + markdownContext: "", + markdownNeedsReload: true, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Mock vault to return our initial cache + mockApp.vault.adapter.exists.mockResolvedValue(true); + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache)); + + // Update Web URL + const testUrl = "https://example.com/test"; + const testContent = "Example web content"; + await projectContextCache.updateWebUrl(mockProject, testUrl, testContent); + + // Verify write was called + expect(mockApp.vault.adapter.write).toHaveBeenCalled(); + + // Reset mock to return the updated cache + const updatedCache = { + ...initialCache, + webContexts: { [testUrl]: testContent }, + }; + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(updatedCache)); + + // Add another URL + const testUrl2 = "https://example.com/test2"; + const testContent2 = "Another example"; + await projectContextCache.updateWebUrl(mockProject, testUrl2, testContent2); + + // Remove URL + const urlsToRemove = [testUrl]; + await projectContextCache.removeWebUrls(mockProject, urlsToRemove); + + // Final cache should only contain the second URL + const finalCache = { + ...initialCache, + webContexts: { [testUrl2]: testContent2 }, + }; + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(finalCache)); + + // Get cache check results + const resultCache = await projectContextCache.get(mockProject); + expect(resultCache).toBeDefined(); + expect(resultCache?.webContexts[testUrl]).toBeUndefined(); + expect(resultCache?.webContexts[testUrl2]).toBe(testContent2); + }); + + test("should update and remove YouTube URLs", async () => { + // Create initial context cache + const initialCache = { + markdownContext: "", + markdownNeedsReload: true, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Mock vault to return our initial cache + mockApp.vault.adapter.exists.mockResolvedValue(true); + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache)); + + // Update YouTube URL + const testYoutubeUrl = "https://youtube.com/watch?v=test123"; + const testYoutubeContent = "Test YouTube transcript"; + await projectContextCache.updateYoutubeUrl(mockProject, testYoutubeUrl, testYoutubeContent); + + // Verify write was called + expect(mockApp.vault.adapter.write).toHaveBeenCalled(); + + // Reset mock to return the updated cache + const updatedCache = { + ...initialCache, + youtubeContexts: { [testYoutubeUrl]: testYoutubeContent }, + }; + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(updatedCache)); + + // Add another URL + const testYoutubeUrl2 = "https://youtube.com/watch?v=test456"; + const testYoutubeContent2 = "Another YouTube transcript"; + await projectContextCache.updateYoutubeUrl(mockProject, testYoutubeUrl2, testYoutubeContent2); + + // Remove URL + const urlsToRemove = [testYoutubeUrl]; + await projectContextCache.removeYoutubeUrls(mockProject, urlsToRemove); + + // Final cache should only contain the second URL + const finalCache = { + ...initialCache, + youtubeContexts: { [testYoutubeUrl2]: testYoutubeContent2 }, + }; + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(finalCache)); + + // Get cache check results + const resultCache = await projectContextCache.get(mockProject); + expect(resultCache).toBeDefined(); + expect(resultCache?.youtubeContexts[testYoutubeUrl]).toBeUndefined(); + expect(resultCache?.youtubeContexts[testYoutubeUrl2]).toBe(testYoutubeContent2); + }); + + test("should safely update cache with updateCacheSafely", async () => { + // Create initial context cache + const initialCache = { + markdownContext: "Initial markdown content", + markdownNeedsReload: false, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Mock cache existence + mockApp.vault.adapter.exists.mockResolvedValue(true); + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache)); + + // Define update function + const updateFn = (cache: any) => { + cache.markdownContext = "Updated markdown content"; + cache.markdownNeedsReload = true; + return cache; + }; + + // Execute safe update + await projectContextCache.updateCacheSafely(mockProject, updateFn); + + // Verify write was called + expect(mockApp.vault.adapter.write).toHaveBeenCalled(); + + // Check that written content contains updated values + const writeCall = mockApp.vault.adapter.write.mock.calls[0]; + const writtenContent = JSON.parse(writeCall[1]); + expect(writtenContent.markdownContext).toBe("Updated markdown content"); + expect(writtenContent.markdownNeedsReload).toBe(true); + }); + + test("should safely update cache with updateCacheSafelyAsync", async () => { + // Create initial context cache + const initialCache = { + markdownContext: "Initial markdown content", + markdownNeedsReload: false, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Mock cache existence + mockApp.vault.adapter.exists.mockResolvedValue(true); + mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache)); + + // Define async update function + const asyncUpdateFn = async (cache: any) => { + // Simulate async operation + await new Promise((resolve) => setTimeout(resolve, 10)); + cache.markdownContext = "Async updated content"; + cache.webContexts = { "https://example.com": "Async web content" }; + return cache; + }; + + // Execute async safe update + await projectContextCache.updateCacheSafelyAsync(mockProject, asyncUpdateFn); + + // Verify write was called + expect(mockApp.vault.adapter.write).toHaveBeenCalled(); + + // Check that written content contains updated values + const writeCall = mockApp.vault.adapter.write.mock.calls[0]; + const writtenContent = JSON.parse(writeCall[1]); + expect(writtenContent.markdownContext).toBe("Async updated content"); + expect(writtenContent.webContexts["https://example.com"]).toBe("Async web content"); + }); + + test("should handle skipIfEmpty parameter correctly in updateCacheSafely", async () => { + // Create a new project instance to ensure cache isolation + const isolatedProject = { + id: "isolated-test-project-id", + name: "Isolated Test Project", + contextSource: { + inclusions: "**/*.md, **/*.pdf", + exclusions: "", + }, + } as any; + + // Mock cache non-existence for this isolated project + mockApp.vault.adapter.exists.mockImplementation((path) => { + if (path.includes("isolated-test-project-id")) { + return Promise.resolve(false); + } + return Promise.resolve(true); + }); + + // Define update function + const updateFn = jest.fn((cache: any) => { + cache.markdownContext = "Should not be called"; + return cache; + }); + + // Execute safe update with skipIfEmpty=true + await projectContextCache.updateCacheSafely(isolatedProject, updateFn, true); + + // Verify updateFn was not called + expect(updateFn).not.toHaveBeenCalled(); + + // Should throw error when skipIfEmpty=false + await expect( + projectContextCache.updateCacheSafely(isolatedProject, updateFn, false) + ).rejects.toThrow(); + }); + + test("should handle concurrent updates with updateCacheSafely", async () => { + // Create initial context cache with proper types + const initialCache: ContextCache = { + markdownContext: "Initial markdown content", + markdownNeedsReload: false, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Mock cache storage to simulate real persistence + let currentCache = initialCache; + mockApp.vault.adapter.exists.mockResolvedValue(true); + mockApp.vault.adapter.read.mockImplementation(() => { + return Promise.resolve(JSON.stringify(currentCache)); + }); + mockApp.vault.adapter.write.mockImplementation((path, content) => { + currentCache = JSON.parse(content); + return Promise.resolve(); + }); + + // Track execution order and update timing + const executionOrder: string[] = []; + const startTime = Date.now(); + + // Create update functions to modify different parts of the cache + const updateMarkdown = (cache: any) => { + executionOrder.push(`markdown-${Date.now() - startTime}`); + cache.markdownContext = "Updated markdown content"; + return cache; + }; + + const updateWeb = (cache: any) => { + executionOrder.push(`web-${Date.now() - startTime}`); + cache.webContexts = { "https://example.com": "Web content" }; + return cache; + }; + + const updateYoutube = (cache: any) => { + executionOrder.push(`youtube-${Date.now() - startTime}`); + cache.youtubeContexts = { "https://youtube.com/test": "YouTube content" }; + return cache; + }; + + // Execute concurrent updates + await Promise.all([ + projectContextCache.updateCacheSafely(mockProject, updateMarkdown), + projectContextCache.updateCacheSafely(mockProject, updateWeb), + projectContextCache.updateCacheSafely(mockProject, updateYoutube), + ]); + + // Verify write was called three times + expect(mockApp.vault.adapter.write).toHaveBeenCalledTimes(3); + + // Verify that all update functions were executed + expect(executionOrder.length).toBe(3); + + // Verify all updates were applied to the final cache + expect(currentCache.markdownContext).toBe("Updated markdown content"); + expect(currentCache.webContexts && currentCache.webContexts["https://example.com"]).toBe( + "Web content" + ); + expect( + currentCache.youtubeContexts && currentCache.youtubeContexts["https://youtube.com/test"] + ).toBe("YouTube content"); + }); + + test("should handle concurrent updates with updateCacheSafelyAsync", async () => { + // Create initial context cache with proper types + const initialCache: ContextCache = { + markdownContext: "Initial markdown content", + markdownNeedsReload: false, + webContexts: {}, + youtubeContexts: {}, + fileContexts: {}, + timestamp: Date.now(), + }; + + // Mock cache storage to simulate real persistence + let currentCache = initialCache; + mockApp.vault.adapter.exists.mockResolvedValue(true); + mockApp.vault.adapter.read.mockImplementation(() => { + return Promise.resolve(JSON.stringify(currentCache)); + }); + + // Track execution order, write calls and timing + const executionOrder: string[] = []; + const writeOrder: string[] = []; + const startTime = Date.now(); + + mockApp.vault.adapter.write.mockImplementation((path, content) => { + const parsed = JSON.parse(content); + currentCache = parsed; + + // Track what was written and when + if (parsed.markdownContext === "Async updated markdown") { + writeOrder.push(`markdown-${Date.now() - startTime}`); + } else if ( + parsed.webContexts && + parsed.webContexts["https://example.com"] === "Async web content" + ) { + writeOrder.push(`web-${Date.now() - startTime}`); + } else if ( + parsed.youtubeContexts && + parsed.youtubeContexts["https://youtube.com/test"] === "Async YouTube content" + ) { + writeOrder.push(`youtube-${Date.now() - startTime}`); + } + + return Promise.resolve(); + }); + + // Create async update functions with different delays + const updateMarkdownAsync = async (cache: any) => { + const startMark = Date.now() - startTime; + executionOrder.push(`markdown-start-${startMark}`); + await new Promise((resolve) => setTimeout(resolve, 30)); + cache.markdownContext = "Async updated markdown"; + executionOrder.push(`markdown-end-${Date.now() - startTime}`); + return cache; + }; + + const updateWebAsync = async (cache: any) => { + const startMark = Date.now() - startTime; + executionOrder.push(`web-start-${startMark}`); + await new Promise((resolve) => setTimeout(resolve, 10)); + cache.webContexts = { "https://example.com": "Async web content" }; + executionOrder.push(`web-end-${Date.now() - startTime}`); + return cache; + }; + + const updateYoutubeAsync = async (cache: any) => { + const startMark = Date.now() - startTime; + executionOrder.push(`youtube-start-${startMark}`); + await new Promise((resolve) => setTimeout(resolve, 20)); + cache.youtubeContexts = { "https://youtube.com/test": "Async YouTube content" }; + executionOrder.push(`youtube-end-${Date.now() - startTime}`); + return cache; + }; + + // Execute concurrent async updates + await Promise.all([ + projectContextCache.updateCacheSafelyAsync(mockProject, updateMarkdownAsync), + projectContextCache.updateCacheSafelyAsync(mockProject, updateWebAsync), + projectContextCache.updateCacheSafelyAsync(mockProject, updateYoutubeAsync), + ]); + + // Verify all writes occurred + expect(mockApp.vault.adapter.write).toHaveBeenCalledTimes(3); + expect(writeOrder.length).toBe(3); + + // Verify all updates were applied to the final cache + expect(currentCache.markdownContext).toBe("Async updated markdown"); + expect(currentCache.webContexts && currentCache.webContexts["https://example.com"]).toBe( + "Async web content" + ); + expect( + currentCache.youtubeContexts && currentCache.youtubeContexts["https://youtube.com/test"] + ).toBe("Async YouTube content"); + + // Log order details for debugging + // console.log("Execution order:", executionOrder); + // console.log("Write order:", writeOrder); + }); }); diff --git a/src/tools/FileParserManager.ts b/src/tools/FileParserManager.ts index 9ce43a15..0f90deac 100644 --- a/src/tools/FileParserManager.ts +++ b/src/tools/FileParserManager.ts @@ -210,7 +210,7 @@ export class Docs4LLMParser implements FileParser { throw new Error("No project context provided for file parsing"); } - const cachedContent = await this.projectContextCache.getFileContext( + const cachedContent = await this.projectContextCache.getOrReuseFileContext( this.currentProject, file.path ); diff --git a/tailwind.config.js b/tailwind.config.js index e58ec7e7..4f372b03 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -28,6 +28,7 @@ module.exports = { "on-accent-inverted": "var(--text-on-accent-inverted)", success: "var(--text-success)", warning: "var(--text-warning)", + loading: "var(--color-blue)", error: "var(--text-error)", accent: "var(--text-accent)", "accent-hover": "var(--text-accent-hover)",