mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
feat: Add project new context loading ui. (#1695)
This commit is contained in:
Emt-lin
GitHub
parent
eca41a929b
commit
fe26c8943b
20 changed files with 2064 additions and 322 deletions
63
package-lock.json
generated
63
package-lock.json
generated
|
|
@ -32,6 +32,7 @@
|
|||
"@radix-ui/react-dropdown-menu": "^2.1.4",
|
||||
"@radix-ui/react-label": "^2.1.0",
|
||||
"@radix-ui/react-popover": "^1.1.4",
|
||||
"@radix-ui/react-progress": "^1.1.7",
|
||||
"@radix-ui/react-scroll-area": "^1.2.9",
|
||||
"@radix-ui/react-select": "^2.1.2",
|
||||
"@radix-ui/react-separator": "^1.1.7",
|
||||
|
|
@ -6329,6 +6330,68 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@radix-ui/react-progress": {
|
||||
"version": "1.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.7.tgz",
|
||||
"integrity": "sha512-vPdg/tF6YC/ynuBIJlk1mm7Le0VgW6ub6J2UWnTQ7/D23KXcPI1qy+0vBkgKgd38RCMJavBXpB83HPNFMTb0Fg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@radix-ui/react-context": "1.1.2",
|
||||
"@radix-ui/react-primitive": "2.1.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/react": "*",
|
||||
"@types/react-dom": "*",
|
||||
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
|
||||
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/react": {
|
||||
"optional": true
|
||||
},
|
||||
"@types/react-dom": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@radix-ui/react-progress/node_modules/@radix-ui/react-context": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.2.tgz",
|
||||
"integrity": "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==",
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@types/react": "*",
|
||||
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/react": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@radix-ui/react-progress/node_modules/@radix-ui/react-primitive": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.3.tgz",
|
||||
"integrity": "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@radix-ui/react-slot": "1.2.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/react": "*",
|
||||
"@types/react-dom": "*",
|
||||
"react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
|
||||
"react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/react": {
|
||||
"optional": true
|
||||
},
|
||||
"@types/react-dom": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@radix-ui/react-roving-focus": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.1.tgz",
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@
|
|||
"@radix-ui/react-dropdown-menu": "^2.1.4",
|
||||
"@radix-ui/react-label": "^2.1.0",
|
||||
"@radix-ui/react-popover": "^1.1.4",
|
||||
"@radix-ui/react-progress": "^1.1.7",
|
||||
"@radix-ui/react-scroll-area": "^1.2.9",
|
||||
"@radix-ui/react-select": "^2.1.2",
|
||||
"@radix-ui/react-separator": "^1.1.7",
|
||||
|
|
|
|||
279
src/LLMProviders/projectLoadTracker.ts
Normal file
279
src/LLMProviders/projectLoadTracker.ts
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
import {
|
||||
FailedItem,
|
||||
ProjectConfig,
|
||||
projectContextLoadAtom,
|
||||
setProjectContextLoadState,
|
||||
updateProjectContextLoadState,
|
||||
} from "@/aiParams";
|
||||
import { ContextCache } from "@/cache/projectContextCache";
|
||||
import { logInfo } from "@/logger";
|
||||
import { settingsStore } from "@/settings/model";
|
||||
import { err2String } from "@/utils";
|
||||
import { App, TFile } from "obsidian";
|
||||
import { isRateLimitError } from "@/utils/rateLimitUtils";
|
||||
|
||||
/**
|
||||
* ProjectLoadTracker is responsible for managing the progress tracking of project file processing
|
||||
*/
|
||||
export class ProjectLoadTracker {
|
||||
private static instance: ProjectLoadTracker;
|
||||
private app: App;
|
||||
|
||||
private constructor(app: App) {
|
||||
this.app = app;
|
||||
}
|
||||
|
||||
public static getInstance(app: App): ProjectLoadTracker {
|
||||
if (!ProjectLoadTracker.instance) {
|
||||
ProjectLoadTracker.instance = new ProjectLoadTracker(app);
|
||||
}
|
||||
return ProjectLoadTracker.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all project context loading states
|
||||
*/
|
||||
public clearAllLoadStates(): void {
|
||||
setProjectContextLoadState({
|
||||
success: [],
|
||||
failed: [],
|
||||
processingFiles: [],
|
||||
total: [],
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an operation and track its execution status
|
||||
*/
|
||||
public async executeWithProcessTracking<T>(
|
||||
key: string,
|
||||
type: FailedItem["type"],
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> {
|
||||
this.setFileOrUrlStartProcess(key);
|
||||
try {
|
||||
const result = await operation();
|
||||
this.setFileOrUrlProcessSuccessful(key);
|
||||
return result;
|
||||
} catch (error) {
|
||||
const errorMessage = isRateLimitError(error)
|
||||
? "Rate limit exceeded. (Rate limit: 50 files or 100MB per 3 hours, whichever is reached first)"
|
||||
: err2String(error);
|
||||
|
||||
this.setFileOrUrlProcessFailed(key, type, errorMessage);
|
||||
throw error; // throw error to outer layer
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a file or URL as processing started
|
||||
*/
|
||||
private setFileOrUrlStartProcess(key: string): void {
|
||||
settingsStore.set(projectContextLoadAtom, (prev) => {
|
||||
const newState = { ...prev };
|
||||
|
||||
// note: we remove the failed file from the failed list when it starts processing
|
||||
if (newState.failed.find((item) => item.path === key)) {
|
||||
newState.failed = newState.failed.filter((file) => file.path !== key);
|
||||
}
|
||||
|
||||
// note: we remove the success file from the success list when it starts processing
|
||||
// For the case where the file cacheKey still exists, but the actual cached content is missing
|
||||
if (newState.success.includes(key)) {
|
||||
newState.success = newState.success.filter((file) => file !== key);
|
||||
}
|
||||
|
||||
// Add to processing files list
|
||||
if (!newState.processingFiles.includes(key)) {
|
||||
newState.processingFiles = [...newState.processingFiles, key];
|
||||
}
|
||||
|
||||
// Ensure file is in the total list
|
||||
if (!newState.total.includes(key)) {
|
||||
newState.total = [...newState.total, key];
|
||||
}
|
||||
|
||||
return newState;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a process as successful
|
||||
*/
|
||||
private setFileOrUrlProcessSuccessful(key: string): void {
|
||||
updateProjectContextLoadState("processingFiles", (prev) => prev.filter((file) => file !== key));
|
||||
updateProjectContextLoadState("success", (prev) => {
|
||||
if (!prev.includes(key)) {
|
||||
return [...prev, key];
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a process as failed
|
||||
*/
|
||||
private setFileOrUrlProcessFailed(key: string, type: FailedItem["type"], error?: string): void {
|
||||
updateProjectContextLoadState("processingFiles", (prev) => prev.filter((file) => file !== key));
|
||||
|
||||
updateProjectContextLoadState("failed", (prev) => {
|
||||
const existingFailed = prev.find((item) => item.path === key);
|
||||
if (!existingFailed) {
|
||||
const failedItem: FailedItem = {
|
||||
path: key,
|
||||
type,
|
||||
error,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
return [...prev, failedItem];
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-compute all items that need to be processed in the project
|
||||
*/
|
||||
public preComputeAllItems(project: ProjectConfig, projectAllFiles: TFile[]): void {
|
||||
logInfo(`[preComputeAllItems] Starting pre-computation for project: ${project.name}`);
|
||||
|
||||
const allItems: string[] = [];
|
||||
|
||||
// 1. Count all matching files (markdown and non-markdown)
|
||||
// Add all matching file paths to the list
|
||||
allItems.push(...projectAllFiles.map((file: TFile) => file.path));
|
||||
|
||||
// 2. Count all Web URLs
|
||||
const configuredWebUrls = project.contextSource?.webUrls?.trim() || "";
|
||||
if (configuredWebUrls) {
|
||||
const webUrls = configuredWebUrls.split("\n").filter((url) => url.trim());
|
||||
allItems.push(...webUrls);
|
||||
}
|
||||
|
||||
// 3. Count all YouTube URLs
|
||||
const configuredYoutubeUrls = project.contextSource?.youtubeUrls?.trim() || "";
|
||||
if (configuredYoutubeUrls) {
|
||||
const youtubeUrls = configuredYoutubeUrls.split("\n").filter((url) => url.trim());
|
||||
allItems.push(...youtubeUrls);
|
||||
}
|
||||
|
||||
// Add all items to the total list
|
||||
if (allItems.length > 0) {
|
||||
const uniqueItems = [...new Set([...allItems])];
|
||||
updateProjectContextLoadState("total", (_) => uniqueItems);
|
||||
logInfo(
|
||||
`[preComputeAllItems] Project ${project.name}: Added ${allItems.length} items to tracking (${uniqueItems.length} total unique items)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark all cached items(besides Non-markdown files) as successful
|
||||
*/
|
||||
public markAllCachedItemsAsSuccess(
|
||||
project: ProjectConfig,
|
||||
contextCache: ContextCache,
|
||||
projectAllFiles: TFile[]
|
||||
): void {
|
||||
logInfo(`[markAllCachedItemsAsSuccess] Starting for project: ${project.name || "default"}`);
|
||||
|
||||
// 1. Mark cached Web URLs
|
||||
const configuredWebUrls = project.contextSource?.webUrls?.trim() || "";
|
||||
if (configuredWebUrls) {
|
||||
const urlsInConfig = configuredWebUrls.split("\n").filter((url) => url.trim());
|
||||
const cachedUrls = urlsInConfig.filter((url) => contextCache.webContexts[url]);
|
||||
cachedUrls.forEach((url) => {
|
||||
this.markCachedItemAsSuccess(url);
|
||||
});
|
||||
if (cachedUrls.length > 0) {
|
||||
logInfo(
|
||||
`[markAllCachedItemsAsSuccess] Project ${project.name}: Marked ${cachedUrls.length} cached Web URLs as successful`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Mark cached YouTube URLs
|
||||
const configuredYoutubeUrls = project.contextSource?.youtubeUrls?.trim() || "";
|
||||
if (configuredYoutubeUrls) {
|
||||
const urlsInConfig = configuredYoutubeUrls.split("\n").filter((url) => url.trim());
|
||||
const cachedUrls = urlsInConfig.filter((url) => contextCache.youtubeContexts[url]);
|
||||
cachedUrls.forEach((url) => {
|
||||
this.markCachedItemAsSuccess(url);
|
||||
});
|
||||
if (cachedUrls.length > 0) {
|
||||
logInfo(
|
||||
`[markAllCachedItemsAsSuccess] Project ${project.name}: Marked ${cachedUrls.length} cached YouTube URLs as successful`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Only mark markdown files present in fileContexts as successful, does not include Non-markdown files.
|
||||
// because track Non-markdown in the processNonMarkdownFiles method
|
||||
if (contextCache.fileContexts) {
|
||||
// only for markdown files
|
||||
const matchingFilesSet = new Set(
|
||||
projectAllFiles.filter((file) => file.extension === "md").map((file: TFile) => file.path)
|
||||
);
|
||||
|
||||
const cachedFilesToMark = Object.keys(contextCache.fileContexts).filter((filePath) =>
|
||||
matchingFilesSet.has(filePath)
|
||||
);
|
||||
|
||||
cachedFilesToMark.forEach((filePath) => {
|
||||
this.markCachedItemAsSuccess(filePath);
|
||||
});
|
||||
|
||||
if (cachedFilesToMark.length > 0) {
|
||||
logInfo(
|
||||
`[markAllCachedItemsAsSuccess] Project ${project.name}: Marked ${
|
||||
cachedFilesToMark.length
|
||||
} cached files that match current project patterns as successful.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a cached item as successful
|
||||
*/
|
||||
public markCachedItemAsSuccess(key: string): void {
|
||||
updateProjectContextLoadState("total", (prev) => {
|
||||
if (!prev.includes(key)) {
|
||||
return [...prev, key];
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
|
||||
// Mark as successful directly
|
||||
updateProjectContextLoadState("success", (prev) => {
|
||||
if (!prev.includes(key)) {
|
||||
return [...prev, key];
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
}
|
||||
|
||||
public makeItemFailed(key: string, type: FailedItem["type"], error?: string): void {
|
||||
updateProjectContextLoadState("total", (prev) => {
|
||||
if (!prev.includes(key)) {
|
||||
return [...prev, key];
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
|
||||
// Check if this item is already in the failed list
|
||||
updateProjectContextLoadState("failed", (prev) => {
|
||||
const existingFailed = prev.find((item) => item.path === key);
|
||||
if (!existingFailed) {
|
||||
const failedItem: FailedItem = {
|
||||
path: key,
|
||||
type,
|
||||
error,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
return [...prev, failedItem];
|
||||
}
|
||||
return prev;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import {
|
||||
FailedItem,
|
||||
getChainType,
|
||||
isProjectMode,
|
||||
ProjectConfig,
|
||||
|
|
@ -23,6 +24,7 @@ import { App, Notice, TFile } from "obsidian";
|
|||
import VectorStoreManager from "../search/vectorStoreManager";
|
||||
import { BrevilabsClient } from "./brevilabsClient";
|
||||
import ChainManager from "./chainManager";
|
||||
import { ProjectLoadTracker } from "./projectLoadTracker";
|
||||
|
||||
export default class ProjectManager {
|
||||
public static instance: ProjectManager;
|
||||
|
|
@ -32,6 +34,7 @@ export default class ProjectManager {
|
|||
private readonly chainMangerInstance: ChainManager;
|
||||
private readonly projectContextCache: ProjectContextCache;
|
||||
private fileParserManager: FileParserManager;
|
||||
private loadTracker: ProjectLoadTracker;
|
||||
|
||||
private constructor(app: App, vectorStoreManager: VectorStoreManager, plugin: CopilotPlugin) {
|
||||
this.app = app;
|
||||
|
|
@ -45,6 +48,7 @@ export default class ProjectManager {
|
|||
true,
|
||||
null
|
||||
);
|
||||
this.loadTracker = ProjectLoadTracker.getInstance(this.app);
|
||||
|
||||
// Set up subscriptions
|
||||
subscribeToModelKeyChange(async () => {
|
||||
|
|
@ -124,6 +128,8 @@ export default class ProjectManager {
|
|||
|
||||
public async switchProject(project: ProjectConfig | null): Promise<void> {
|
||||
try {
|
||||
// Clear all project context loading states
|
||||
this.loadTracker.clearAllLoadStates();
|
||||
setProjectLoading(true);
|
||||
logInfo("Project loading started...");
|
||||
|
||||
|
|
@ -187,6 +193,10 @@ export default class ProjectManager {
|
|||
}
|
||||
|
||||
private async loadProjectContext(project: ProjectConfig): Promise<ContextCache | null> {
|
||||
// for update context condition
|
||||
this.loadTracker.clearAllLoadStates();
|
||||
setProjectLoading(true);
|
||||
|
||||
try {
|
||||
if (!project.contextSource) {
|
||||
logWarn(`[loadProjectContext] Project ${project.name}: No contextSource. Aborting.`);
|
||||
|
|
@ -194,86 +204,31 @@ export default class ProjectManager {
|
|||
}
|
||||
logInfo(`[loadProjectContext] Starting for project: ${project.name}`);
|
||||
|
||||
const initialProjectCache = await this.projectContextCache.get(project);
|
||||
const contextCache = initialProjectCache || {
|
||||
markdownContext: "",
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
markdownNeedsReload: true,
|
||||
};
|
||||
if (!initialProjectCache) {
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: No existing cache found, building fresh context.`
|
||||
`[loadProjectContext] Project ${project.name}: Cleared all project context load states`
|
||||
);
|
||||
} else {
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Existing cache found. MarkdownNeedsReload: ${contextCache.markdownNeedsReload}`
|
||||
);
|
||||
}
|
||||
|
||||
const contextCache = await this.projectContextCache.getOrInitializeCache(project);
|
||||
|
||||
const projectAllFiles = this.getProjectAllFiles(project);
|
||||
|
||||
// Pre-count all items that need to be processed
|
||||
this.loadTracker.preComputeAllItems(project, projectAllFiles);
|
||||
this.loadTracker.markAllCachedItemsAsSuccess(project, contextCache, projectAllFiles);
|
||||
|
||||
const [updatedContextCacheAfterSources] = await Promise.all([
|
||||
this.processMarkdownFiles(project, contextCache),
|
||||
this.processMarkdownFiles(project, contextCache, projectAllFiles),
|
||||
this.processWebUrls(project, contextCache),
|
||||
this.processYoutubeUrls(project, contextCache),
|
||||
]);
|
||||
|
||||
// After other contexts are processed, ensure all referenced non-markdown files are parsed and cached
|
||||
if (updatedContextCacheAfterSources.fileContexts) {
|
||||
const fileContextCount = Object.keys(updatedContextCacheAfterSources.fileContexts).length;
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Checking ${fileContextCount} fileContexts for non-markdown processing.`
|
||||
);
|
||||
|
||||
if (fileContextCount > 0) {
|
||||
this.fileParserManager = new FileParserManager(
|
||||
BrevilabsClient.getInstance(),
|
||||
this.app.vault,
|
||||
true,
|
||||
project
|
||||
);
|
||||
let processedNonMdCount = 0;
|
||||
for (const filePath in updatedContextCacheAfterSources.fileContexts) {
|
||||
const file = this.app.vault.getAbstractFileByPath(filePath);
|
||||
if (file instanceof TFile && file.extension !== "md") {
|
||||
if (this.fileParserManager.supportsExtension(file.extension)) {
|
||||
try {
|
||||
const existingContent = await this.projectContextCache.getFileContext(
|
||||
project,
|
||||
filePath
|
||||
);
|
||||
if (!existingContent) {
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Parsing/caching new/updated file: ${filePath}`
|
||||
);
|
||||
await this.fileParserManager.parseFile(file, this.app.vault);
|
||||
processedNonMdCount++;
|
||||
} // else { logInfo for skipped can be too verbose }
|
||||
} catch (error) {
|
||||
logError(
|
||||
`[loadProjectContext] Project ${project.name}: Error parsing file ${filePath}:`,
|
||||
error
|
||||
);
|
||||
|
||||
// Check if this is a rate limit error and re-throw it to fail the entire operation
|
||||
if (isRateLimitError(error)) {
|
||||
throw error; // Re-throw to fail the entire operation
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (processedNonMdCount > 0) {
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Processed and cached ${processedNonMdCount} non-markdown files.`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
updatedContextCacheAfterSources.timestamp = Date.now();
|
||||
await this.projectContextCache.set(project, updatedContextCacheAfterSources);
|
||||
// Note: Since non-markdown files cannot pass cache parameters , so we need to save the context cache first
|
||||
await this.projectContextCache.setCacheSafely(project, updatedContextCacheAfterSources);
|
||||
|
||||
// After other contexts are processed, ensure all referenced non-markdown files are parsed and cached
|
||||
await this.processNonMarkdownFiles(project, projectAllFiles);
|
||||
|
||||
logInfo(`[loadProjectContext] Completed for project: ${project.name}.`);
|
||||
return updatedContextCacheAfterSources;
|
||||
} catch (error) {
|
||||
|
|
@ -315,11 +270,10 @@ export default class ProjectManager {
|
|||
const nextUrls = nextWebUrls.split("\n").filter((url) => url.trim());
|
||||
|
||||
// Remove context for URLs that no longer exist
|
||||
for (const url of prevUrls) {
|
||||
if (!nextUrls.includes(url)) {
|
||||
await this.projectContextCache.removeWebUrl(nextProject, url);
|
||||
}
|
||||
}
|
||||
await this.projectContextCache.removeWebUrls(
|
||||
nextProject,
|
||||
prevUrls.filter((url) => !nextUrls.includes(url))
|
||||
);
|
||||
}
|
||||
|
||||
// Check if YouTube URLs configuration has changed
|
||||
|
|
@ -332,11 +286,10 @@ export default class ProjectManager {
|
|||
const nextUrls = nextYoutubeUrls.split("\n").filter((url) => url.trim());
|
||||
|
||||
// Remove context for URLs that no longer exist
|
||||
for (const url of prevUrls) {
|
||||
if (!nextUrls.includes(url)) {
|
||||
await this.projectContextCache.removeYoutubeUrl(nextProject, url);
|
||||
}
|
||||
}
|
||||
await this.projectContextCache.removeYoutubeUrls(
|
||||
nextProject,
|
||||
prevUrls.filter((url) => !nextUrls.includes(url))
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
logError(`Error comparing project configurations: ${error}`);
|
||||
|
|
@ -424,7 +377,7 @@ export default class ProjectManager {
|
|||
|
||||
// Retrieve file content from FileCache
|
||||
const content =
|
||||
(await this.projectContextCache.getFileContext(project, filePath)) ||
|
||||
(await this.projectContextCache.getOrReuseFileContext(project, filePath)) ||
|
||||
"[Content not available]"; // This is expected for files not processed into FileCache
|
||||
|
||||
return `[[${fileName}]]\npath: ${filePath}\ntype: ${fileType}\nmodified: ${new Date(fileContext.timestamp).toISOString()}\n\n${content}`;
|
||||
|
|
@ -497,22 +450,10 @@ ${contextParts.join("\n\n")}
|
|||
|
||||
private async processMarkdownFiles(
|
||||
project: ProjectConfig,
|
||||
contextCache: ContextCache
|
||||
contextCache: ContextCache,
|
||||
projectAllFiles: TFile[]
|
||||
): Promise<ContextCache> {
|
||||
logInfo(`[processMarkdownFiles] Starting for project: ${project.name}`);
|
||||
const initialFileContextsCount = Object.keys(contextCache.fileContexts || {}).length;
|
||||
|
||||
if (project.contextSource?.inclusions || project.contextSource?.exclusions) {
|
||||
contextCache = await this.projectContextCache.updateProjectFilesFromPatterns(
|
||||
project,
|
||||
contextCache
|
||||
);
|
||||
const newFileContextsCount = Object.keys(contextCache.fileContexts || {}).length;
|
||||
if (newFileContextsCount > initialFileContextsCount) {
|
||||
logInfo(
|
||||
`[processMarkdownFiles] Project ${project.name}: Added ${newFileContextsCount - initialFileContextsCount} new file references via updateProjectFilesFromPatterns.`
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
contextCache.markdownNeedsReload ||
|
||||
|
|
@ -520,59 +461,39 @@ ${contextParts.join("\n\n")}
|
|||
!contextCache.markdownContext.trim()
|
||||
) {
|
||||
logInfo(`[processMarkdownFiles] Project ${project.name}: Processing markdown content.`);
|
||||
const markdownContent = await this.processFileContext(
|
||||
project.contextSource.inclusions,
|
||||
project.contextSource.exclusions,
|
||||
project
|
||||
const markdownContent = await this.processMarkdownFileContext(projectAllFiles);
|
||||
|
||||
// add context reference to markdown file
|
||||
this.projectContextCache.updateProjectMarkdownFilesFromPatterns(
|
||||
project,
|
||||
contextCache,
|
||||
projectAllFiles
|
||||
);
|
||||
|
||||
contextCache.markdownContext = markdownContent;
|
||||
contextCache.markdownNeedsReload = false;
|
||||
|
||||
logInfo(`[processMarkdownFiles] Project ${project.name}: Markdown content updated.`);
|
||||
} else {
|
||||
logInfo(
|
||||
`[processMarkdownFiles] Project ${project.name}: Markdown content already up-to-date.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
logInfo(
|
||||
`[processMarkdownFiles] Completed for project: ${project.name}. Total fileContexts: ${Object.keys(contextCache.fileContexts || {}).length}`
|
||||
);
|
||||
return contextCache;
|
||||
}
|
||||
|
||||
private async processFileContext(
|
||||
inclusions?: string,
|
||||
exclusions?: string,
|
||||
project?: ProjectConfig
|
||||
): Promise<string> {
|
||||
if (!inclusions && !exclusions) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!project) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// NOTE: Must not fallback to GLOBAL inclusions and exclusions in Copilot settings in Projects!
|
||||
// This is to avoid project inclusions in the project that conflict with the global ones
|
||||
// Project UI should be the ONLY source of truth for project inclusions and exclusions
|
||||
const { inclusions: inclusionPatterns, exclusions: exclusionPatterns } = getMatchingPatterns({
|
||||
inclusions,
|
||||
exclusions,
|
||||
isProject: true,
|
||||
});
|
||||
|
||||
private async processMarkdownFileContext(projectAllFiles: TFile[]): Promise<string> {
|
||||
// FileParserManager will be used to process these files when they're accessed,
|
||||
// either immediately or on-demand when the context is formatted
|
||||
|
||||
// Get all markdown files that match the inclusion/exclusion patterns
|
||||
// Note: We're only processing markdown files here, other file types
|
||||
// are handled by FileParserManager and stored in the file cache
|
||||
const files = this.app.vault.getFiles().filter((file) => {
|
||||
return (
|
||||
file.extension === "md" && shouldIndexFile(file, inclusionPatterns, exclusionPatterns, true)
|
||||
);
|
||||
});
|
||||
const files = projectAllFiles.filter((file) => file.extension === "md");
|
||||
|
||||
logInfo(`Found ${files.length} markdown files to process for project context`);
|
||||
|
||||
|
|
@ -583,15 +504,25 @@ ${contextParts.join("\n\n")}
|
|||
let metadata = "";
|
||||
|
||||
try {
|
||||
const stat = await this.app.vault.adapter.stat(file.path);
|
||||
// Only process markdown files here
|
||||
const [stat, fileContent] = await this.loadTracker.executeWithProcessTracking(
|
||||
file.path,
|
||||
"md",
|
||||
async () => {
|
||||
return Promise.all([
|
||||
this.app.vault.adapter.stat(file.path),
|
||||
this.app.vault.read(file),
|
||||
]);
|
||||
}
|
||||
);
|
||||
|
||||
metadata = `[[${file.basename}]]
|
||||
path: ${file.path}
|
||||
type: ${file.extension}
|
||||
created: ${stat ? new Date(stat.ctime).toISOString() : "unknown"}
|
||||
modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`;
|
||||
|
||||
// Only process markdown files here
|
||||
content = await this.app.vault.read(file);
|
||||
content = fileContent;
|
||||
logInfo(`Completed processing markdown file: ${file.path}`);
|
||||
} catch (error) {
|
||||
logError(`Error processing file ${file.path}: ${error}`);
|
||||
|
|
@ -650,8 +581,8 @@ modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`;
|
|||
}
|
||||
|
||||
const webContextPromises = urlsToFetch.map(async (url) => {
|
||||
// processWebUrlsContext itself should log errors if a specific URL fetch fails.
|
||||
const webContext = await this.processWebUrlsContext(url);
|
||||
// processWebUrlContext itself should log errors if a specific URL fetch fails.
|
||||
const webContext = await this.processWebUrlContext(url);
|
||||
if (webContext) {
|
||||
logInfo(
|
||||
`[processWebUrls] Project ${project.name}: Successfully fetched content for URL: ${url.substring(0, 50)}...`
|
||||
|
|
@ -717,7 +648,7 @@ modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`;
|
|||
}
|
||||
|
||||
const youtubeContextPromises = urlsToFetch.map(async (url) => {
|
||||
const youtubeContext = await this.processYoutubeUrlsContext(url);
|
||||
const youtubeContext = await this.processYoutubeUrlContext(url);
|
||||
if (youtubeContext) {
|
||||
logInfo(
|
||||
`[processYoutubeUrls] Project ${project.name}: Successfully fetched transcript for YouTube URL: ${url.substring(0, 50)}...`
|
||||
|
|
@ -742,44 +673,252 @@ modified: ${stat ? new Date(stat.mtime).toISOString() : "unknown"}`;
|
|||
return contextCache;
|
||||
}
|
||||
|
||||
private async processWebUrlsContext(webUrls?: string): Promise<string> {
|
||||
if (!webUrls?.trim()) {
|
||||
private async processWebUrlContext(webUrl?: string): Promise<string> {
|
||||
if (!webUrl?.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
try {
|
||||
const mention = Mention.getInstance();
|
||||
const { urlContext } = await mention.processUrls(webUrls);
|
||||
const { urlContext } = await this.loadTracker.executeWithProcessTracking(
|
||||
webUrl,
|
||||
"web",
|
||||
async () => {
|
||||
const result = await mention.processUrls(webUrl);
|
||||
|
||||
if (result.processedErrorUrls[webUrl]) {
|
||||
throw new Error(result.processedErrorUrls[webUrl]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
);
|
||||
return urlContext || "";
|
||||
} catch (error) {
|
||||
logError(`Failed to process web URLs: ${error}`);
|
||||
new Notice(`Failed to process web URLs: ${err2String(error)}`);
|
||||
logError(`Failed to process web URL: ${error}`);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private async processYoutubeUrlsContext(youtubeUrls?: string): Promise<string> {
|
||||
if (!youtubeUrls?.trim()) {
|
||||
private async processYoutubeUrlContext(youtubeUrl?: string): Promise<string> {
|
||||
if (!youtubeUrl?.trim()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const urls = youtubeUrls.split("\n").filter((url) => url.trim());
|
||||
const processPromises = urls.map(async (url) => {
|
||||
try {
|
||||
const response = await BrevilabsClient.getInstance().youtube4llm(url);
|
||||
const response = await this.loadTracker.executeWithProcessTracking(
|
||||
youtubeUrl,
|
||||
"youtube",
|
||||
async () => {
|
||||
return BrevilabsClient.getInstance().youtube4llm(youtubeUrl);
|
||||
}
|
||||
);
|
||||
if (response.response.transcript) {
|
||||
return `\n\nYouTube transcript from ${url}:\n${response.response.transcript}`;
|
||||
return `\n\nYouTube transcript from ${youtubeUrl}:\n${response.response.transcript}`;
|
||||
}
|
||||
return "";
|
||||
} catch (error) {
|
||||
logError(`Failed to process YouTube URL ${url}: ${error}`);
|
||||
new Notice(`Failed to process YouTube URL ${url}: ${err2String(error)}`);
|
||||
logError(`Failed to process YouTube URL ${youtubeUrl}: ${error}`);
|
||||
new Notice(`Failed to process YouTube URL ${youtubeUrl}: ${err2String(error)}`);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private async processNonMarkdownFiles(
|
||||
project: ProjectConfig,
|
||||
projectAllFiles: TFile[]
|
||||
): Promise<void> {
|
||||
const nonMarkdownFiles = projectAllFiles.filter((file) => file.extension !== "md");
|
||||
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Checking for non-markdown processing: ${nonMarkdownFiles.length} files .`
|
||||
);
|
||||
|
||||
if (nonMarkdownFiles.length <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.fileParserManager = new FileParserManager(
|
||||
BrevilabsClient.getInstance(),
|
||||
this.app.vault,
|
||||
true,
|
||||
project
|
||||
);
|
||||
|
||||
let processedNonMdCount = 0;
|
||||
|
||||
for (const file of nonMarkdownFiles) {
|
||||
const filePath = file.path;
|
||||
if (this.fileParserManager.supportsExtension(file.extension)) {
|
||||
try {
|
||||
await this.loadTracker.executeWithProcessTracking(filePath, "nonMd", async () => {
|
||||
const existingContent = await this.projectContextCache.getOrReuseFileContext(
|
||||
project,
|
||||
filePath
|
||||
);
|
||||
if (existingContent) {
|
||||
processedNonMdCount++;
|
||||
} else {
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Parsing/caching new/updated file: ${filePath}`
|
||||
);
|
||||
|
||||
await this.fileParserManager.parseFile(file, this.app.vault);
|
||||
processedNonMdCount++;
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logError(
|
||||
`[loadProjectContext] Project ${project.name}: Error parsing file ${filePath}:`,
|
||||
error
|
||||
);
|
||||
|
||||
// Check if this is a rate limit error and re-throw it to fail the entire operation
|
||||
if (isRateLimitError(error)) {
|
||||
throw error; // Re-throw to fail the entire operation
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (processedNonMdCount > 0) {
|
||||
logInfo(
|
||||
`[loadProjectContext] Project ${project.name}: Processed and cached ${processedNonMdCount} non-markdown files.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry failed item
|
||||
* @param failedItem Failed item information
|
||||
*/
|
||||
public async retryFailedItem(failedItem: FailedItem): Promise<void> {
|
||||
try {
|
||||
if (!this.currentProjectId) {
|
||||
logWarn("[retryFailedItem] No current project, aborting retry");
|
||||
return;
|
||||
}
|
||||
|
||||
const project = getSettings().projectList.find((p) => p.id === this.currentProjectId);
|
||||
if (!project) {
|
||||
logError(`[retryFailedItem] Current project not found: ${this.currentProjectId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
logInfo(`[retryFailedItem] Starting retry for ${failedItem.type} item: ${failedItem.path}`);
|
||||
|
||||
// Handle different retry types
|
||||
switch (failedItem.type) {
|
||||
case "web":
|
||||
await this.retryWebUrl(project, failedItem.path);
|
||||
break;
|
||||
case "youtube":
|
||||
await this.retryYoutubeUrl(project, failedItem.path);
|
||||
break;
|
||||
case "md":
|
||||
await this.retryMarkdownFile(project, failedItem.path);
|
||||
break;
|
||||
case "nonMd":
|
||||
await this.retryNonMarkdownFile(project, failedItem.path);
|
||||
break;
|
||||
default:
|
||||
logWarn(`[retryFailedItem] Unknown item type: ${failedItem.type}`);
|
||||
return;
|
||||
}
|
||||
|
||||
logInfo(`[retryFailedItem] Successfully retried ${failedItem.type} item: ${failedItem.path}`);
|
||||
new Notice(`Retry successful: ${failedItem.path}`);
|
||||
} catch (error) {
|
||||
logError(
|
||||
`[retryFailedItem] Failed to retry ${failedItem.type} item ${failedItem.path}:`,
|
||||
error
|
||||
);
|
||||
new Notice(`Retry failed: ${err2String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async retryWebUrl(project: ProjectConfig, url: string): Promise<void> {
|
||||
const webContext = await this.processWebUrlContext(url);
|
||||
if (!webContext) {
|
||||
logWarn(`[retryWebUrl] Project ${project.name}: Fetched empty content for Web URL: ${url}`);
|
||||
return;
|
||||
}
|
||||
|
||||
logInfo(
|
||||
`[retryWebUrl] Project ${project.name}: Successfully fetched content for URL: ${url.substring(0, 50)}...`
|
||||
);
|
||||
await this.projectContextCache.updateWebUrl(project, url, webContext);
|
||||
}
|
||||
|
||||
private async retryYoutubeUrl(project: ProjectConfig, url: string): Promise<void> {
|
||||
const youtubeContext = await this.processYoutubeUrlContext(url);
|
||||
if (!youtubeContext) {
|
||||
logWarn(
|
||||
`[retryYoutubeUrl] Project ${project.name}: Fetched empty transcript for YouTube URL: ${url}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
logInfo(
|
||||
`[retryYoutubeUrl] Project ${project.name}: Successfully fetched transcript for YouTube URL: ${url.substring(0, 50)}...`
|
||||
);
|
||||
await this.projectContextCache.updateYoutubeUrl(project, url, youtubeContext);
|
||||
}
|
||||
|
||||
private async retryMarkdownFile(project: ProjectConfig, filePath: string): Promise<void> {
|
||||
const file = this.app.vault.getAbstractFileByPath(filePath);
|
||||
if (!(file instanceof TFile) || file.extension !== "md") {
|
||||
throw new Error(`File not found or not a markdown file: ${filePath}`);
|
||||
}
|
||||
|
||||
try {
|
||||
// add flag to track reprocessing of Markdown
|
||||
await this.loadTracker.executeWithProcessTracking(file.path, "md", async () => {});
|
||||
|
||||
logInfo(`[retryMarkdownFile] Successfully reprocessed markdown file: ${filePath}`);
|
||||
|
||||
// flag the markdown context as needing a reload
|
||||
await this.projectContextCache.invalidateMarkdownContext(project);
|
||||
} catch (error) {
|
||||
logError(`[retryMarkdownFile] Error processing file ${filePath}: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async retryNonMarkdownFile(project: ProjectConfig, filePath: string): Promise<void> {
|
||||
const file = this.app.vault.getAbstractFileByPath(filePath);
|
||||
if (!(file instanceof TFile) || file.extension === "md") {
|
||||
throw new Error(`File not found or is a markdown file: ${filePath}`);
|
||||
}
|
||||
|
||||
if (!this.fileParserManager.supportsExtension(file.extension)) {
|
||||
throw new Error(`Unsupported file extension: ${file.extension}`);
|
||||
}
|
||||
|
||||
try {
|
||||
await this.loadTracker.executeWithProcessTracking(filePath, "nonMd", async () => {
|
||||
return this.fileParserManager.parseFile(file, this.app.vault);
|
||||
});
|
||||
|
||||
const results = await Promise.all(processPromises);
|
||||
return results.join("");
|
||||
logInfo(`[retryNonMarkdownFile] Successfully reprocessed non-markdown file: ${filePath}`);
|
||||
} catch (error) {
|
||||
logError(`[retryNonMarkdownFile] Error processing file ${filePath}: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private getProjectAllFiles(project: ProjectConfig) {
|
||||
// NOTE: Must not fallback to GLOBAL inclusions and exclusions in Copilot settings in Projects!
|
||||
// This is to avoid project inclusions in the project that conflict with the global ones
|
||||
// Project UI should be the ONLY source of truth for project inclusions and exclusions
|
||||
const { inclusions: inclusionPatterns, exclusions: exclusionPatterns } = getMatchingPatterns({
|
||||
inclusions: project.contextSource.inclusions,
|
||||
exclusions: project.contextSource.exclusions,
|
||||
isProject: true,
|
||||
});
|
||||
|
||||
return this.app.vault.getFiles().filter((file: TFile) => {
|
||||
return shouldIndexFile(file, inclusionPatterns, exclusionPatterns, true);
|
||||
});
|
||||
}
|
||||
|
||||
public onunload(): void {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,27 @@ const chainTypeAtom = atom(
|
|||
const currentProjectAtom = atom<ProjectConfig | null>(null);
|
||||
const projectLoadingAtom = atom<boolean>(false);
|
||||
|
||||
export interface FailedItem {
|
||||
path: string;
|
||||
type: "md" | "web" | "youtube" | "nonMd";
|
||||
error?: string;
|
||||
timestamp?: number;
|
||||
}
|
||||
|
||||
interface ProjectContextLoadState {
|
||||
success: Array<string>;
|
||||
failed: Array<FailedItem>;
|
||||
processingFiles: Array<string>;
|
||||
total: Array<string>;
|
||||
}
|
||||
|
||||
export const projectContextLoadAtom = atom<ProjectContextLoadState>({
|
||||
success: [],
|
||||
failed: [],
|
||||
processingFiles: [],
|
||||
total: [],
|
||||
});
|
||||
|
||||
const selectedTextContextsAtom = atom<SelectedTextContext[]>([]);
|
||||
|
||||
export interface ProjectConfig {
|
||||
|
|
@ -51,7 +72,7 @@ export interface ProjectConfig {
|
|||
maxTokens?: number;
|
||||
};
|
||||
contextSource: {
|
||||
inclusions: string;
|
||||
inclusions?: string;
|
||||
exclusions?: string;
|
||||
webUrls?: string;
|
||||
youtubeUrls?: string;
|
||||
|
|
@ -239,3 +260,50 @@ export function useSelectedTextContexts() {
|
|||
store: settingsStore,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the project context load state from the atom.
|
||||
*/
|
||||
export function getProjectContextLoadState(): Readonly<ProjectContextLoadState> {
|
||||
return settingsStore.get(projectContextLoadAtom);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the project context load state in the atom.
|
||||
*/
|
||||
export function setProjectContextLoadState(state: ProjectContextLoadState) {
|
||||
settingsStore.set(projectContextLoadAtom, state);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates a specific field in the project context load state.
|
||||
*/
|
||||
export function updateProjectContextLoadState<K extends keyof ProjectContextLoadState>(
|
||||
key: K,
|
||||
valueFn: (prev: ProjectContextLoadState[K]) => ProjectContextLoadState[K]
|
||||
) {
|
||||
settingsStore.set(projectContextLoadAtom, (prev) => ({
|
||||
...prev,
|
||||
[key]: valueFn(prev[key]),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribes to changes in the project context load state.
|
||||
*/
|
||||
export function subscribeToProjectContextLoadChange(
|
||||
callback: (state: ProjectContextLoadState) => void
|
||||
): () => void {
|
||||
return settingsStore.sub(projectContextLoadAtom, () => {
|
||||
callback(settingsStore.get(projectContextLoadAtom));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to get the project context load state from the atom.
|
||||
*/
|
||||
export function useProjectContextLoad() {
|
||||
return useAtom(projectContextLoadAtom, {
|
||||
store: settingsStore,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
432
src/cache/projectContextCache.ts
vendored
432
src/cache/projectContextCache.ts
vendored
|
|
@ -6,6 +6,7 @@ import { getSettings } from "@/settings/model";
|
|||
import { MD5 } from "crypto-js";
|
||||
import { TAbstractFile, TFile, Vault } from "obsidian";
|
||||
import debounce from "lodash.debounce";
|
||||
import { Mutex } from "async-mutex";
|
||||
|
||||
export interface ContextCache {
|
||||
// Markdown context
|
||||
|
|
@ -38,6 +39,8 @@ export class ProjectContextCache {
|
|||
private vault: Vault;
|
||||
private fileCache: FileCache<string>;
|
||||
private static readonly DEBOUNCE_DELAY = 5000; // 5 seconds
|
||||
private projectMutexMap: Map<string, Mutex> = new Map();
|
||||
private mutexCreationMutex: Mutex = new Mutex(); // Global lock to protect project mutex creation
|
||||
|
||||
private constructor() {
|
||||
this.vault = app.vault;
|
||||
|
|
@ -65,6 +68,9 @@ export class ProjectContextCache {
|
|||
this.vault.off("modify", this.handleFileEvent);
|
||||
this.vault.off("delete", this.handleFileEvent);
|
||||
this.vault.off("rename", this.handleFileEvent);
|
||||
|
||||
// Clean up project mutexes
|
||||
this.projectMutexMap.clear();
|
||||
}
|
||||
|
||||
private initializeEventListeners() {
|
||||
|
|
@ -143,6 +149,32 @@ export class ProjectContextCache {
|
|||
return `${this.cacheDir}/${cacheKey}.json`;
|
||||
}
|
||||
|
||||
private async getOrCreateProjectMutex(project: ProjectConfig): Promise<Mutex> {
|
||||
const projectId = project.id;
|
||||
|
||||
// Quick check without lock for performance
|
||||
const existingMutex = this.projectMutexMap.get(projectId);
|
||||
if (existingMutex) {
|
||||
return existingMutex;
|
||||
}
|
||||
|
||||
// Use global lock to ensure atomic creation
|
||||
return await this.mutexCreationMutex.runExclusive(async () => {
|
||||
// Double-check inside the lock
|
||||
const mutex = this.projectMutexMap.get(projectId);
|
||||
if (mutex) {
|
||||
return mutex;
|
||||
}
|
||||
|
||||
// Create new mutex safely
|
||||
const newMutex = new Mutex();
|
||||
this.projectMutexMap.set(projectId, newMutex);
|
||||
logInfo(`Created new mutex for project: ${project.name} (ID: ${projectId})`);
|
||||
|
||||
return newMutex;
|
||||
});
|
||||
}
|
||||
|
||||
async get(project: ProjectConfig): Promise<ContextCache | null> {
|
||||
try {
|
||||
const cacheKey = this.getCacheKey(project);
|
||||
|
|
@ -171,6 +203,25 @@ export class ProjectContextCache {
|
|||
}
|
||||
}
|
||||
|
||||
async getOrInitializeCache(project: ProjectConfig): Promise<ContextCache> {
|
||||
const initialProjectCache = await this.get(project);
|
||||
|
||||
if (initialProjectCache) {
|
||||
logInfo(
|
||||
`[getOrInitializeCache] Project ${project.name}: Existing cache found. MarkdownNeedsReload: ${initialProjectCache.markdownNeedsReload}`
|
||||
);
|
||||
return initialProjectCache;
|
||||
}
|
||||
|
||||
logInfo(
|
||||
`[getOrInitializeCache] Project ${project.name}: No existing cache found, building fresh context.`
|
||||
);
|
||||
|
||||
const newCache = this.createEmptyCache();
|
||||
await this.setWithoutMutex(project, newCache);
|
||||
return newCache;
|
||||
}
|
||||
|
||||
getSync(project: ProjectConfig): ContextCache | null {
|
||||
try {
|
||||
const cacheKey = this.getCacheKey(project);
|
||||
|
|
@ -187,18 +238,37 @@ export class ProjectContextCache {
|
|||
}
|
||||
}
|
||||
|
||||
async set(project: ProjectConfig, contextCache: ContextCache): Promise<void> {
|
||||
private async set(project: ProjectConfig, contextCache: ContextCache): Promise<void> {
|
||||
const mutex = await this.getOrCreateProjectMutex(project);
|
||||
|
||||
if (mutex.isLocked()) {
|
||||
logInfo(`Waiting for project cache lock for project: ${project.name}`);
|
||||
}
|
||||
|
||||
return await mutex.runExclusive(async () => {
|
||||
logInfo(`Acquired cache lock for project: ${project.name}`);
|
||||
return await this.setWithoutMutex(project, contextCache);
|
||||
});
|
||||
}
|
||||
|
||||
private async setWithoutMutex(project: ProjectConfig, contextCache: ContextCache): Promise<void> {
|
||||
try {
|
||||
await this.ensureCacheDir();
|
||||
const cacheKey = this.getCacheKey(project);
|
||||
const cachePath = this.getCachePath(cacheKey);
|
||||
logInfo("Caching context for project:", project.name);
|
||||
|
||||
// Create a deep copy to avoid reference issues
|
||||
const contextCacheCopy = JSON.parse(JSON.stringify(contextCache));
|
||||
|
||||
// Store in memory cache
|
||||
this.memoryCache.set(cacheKey, contextCache);
|
||||
this.memoryCache.set(cacheKey, contextCacheCopy);
|
||||
|
||||
// Store in file cache
|
||||
await this.vault.adapter.write(cachePath, JSON.stringify(contextCache));
|
||||
await this.vault.adapter.write(cachePath, JSON.stringify(contextCacheCopy));
|
||||
} catch (error) {
|
||||
logError("Error writing to project context cache:", error);
|
||||
throw error; // Re-throw to maintain error propagation
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -209,7 +279,7 @@ export class ProjectContextCache {
|
|||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
markdownNeedsReload: false,
|
||||
markdownNeedsReload: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -314,6 +384,10 @@ export class ProjectContextCache {
|
|||
`[clearForProject] Project ${project.name}: Main project cache file not found (already deleted or never existed): ${cachePath}`
|
||||
);
|
||||
}
|
||||
// Clean up the mutex for this project to prevent memory leaks
|
||||
this.projectMutexMap.delete(project.id);
|
||||
logInfo(`[clearForProject] Cleaned up mutex for project: ${project.name}`);
|
||||
|
||||
logInfo(`[clearForProject] Completed for project: ${project.name}`);
|
||||
} catch (error) {
|
||||
logError(`[clearForProject] Error for project ${project.name} (ID: ${project.id}):`, error);
|
||||
|
|
@ -332,8 +406,9 @@ export class ProjectContextCache {
|
|||
project: ProjectConfig,
|
||||
forceReloadAllRemotes: boolean = false
|
||||
): Promise<void> {
|
||||
const cache = await this.get(project);
|
||||
if (cache) {
|
||||
await this.updateCacheSafely(
|
||||
project,
|
||||
(cache) => {
|
||||
cache.markdownContext = "";
|
||||
cache.markdownNeedsReload = true;
|
||||
|
||||
|
|
@ -343,36 +418,37 @@ export class ProjectContextCache {
|
|||
logInfo(`Flagged Web/YouTube contexts for full reload for project ${project.name}`);
|
||||
}
|
||||
|
||||
await this.set(project, cache);
|
||||
|
||||
// Also clean up any file references that no longer match the project's patterns
|
||||
await this.cleanupProjectFileReferences(project);
|
||||
const cleanedCache = this.cleanupFileReferencesInCache(project, cache);
|
||||
|
||||
logInfo(`Invalidated markdown context for project ${project.name}`);
|
||||
}
|
||||
return cleanedCache;
|
||||
},
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the markdown context for a project
|
||||
*/
|
||||
async updateMarkdownContext(project: ProjectConfig, content: string): Promise<void> {
|
||||
const cache = (await this.get(project)) || this.createEmptyCache();
|
||||
return await this.updateCacheSafely(project, (cache) => {
|
||||
cache.markdownContext = content;
|
||||
cache.markdownNeedsReload = false;
|
||||
await this.set(project, cache);
|
||||
logInfo(`Updated markdown context for project ${project.name}`);
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear only the markdown context for a project
|
||||
*/
|
||||
async clearMarkdownContext(project: ProjectConfig): Promise<void> {
|
||||
const cache = await this.get(project);
|
||||
if (cache) {
|
||||
await this.updateCacheSafely(project, (cache) => {
|
||||
cache.markdownContext = "";
|
||||
cache.markdownNeedsReload = true;
|
||||
await this.set(project, cache);
|
||||
}
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
//===========================================================================
|
||||
|
|
@ -380,10 +456,41 @@ export class ProjectContextCache {
|
|||
// Non-markdown files cached in fileCache after processing by FileParserManager
|
||||
//===========================================================================
|
||||
|
||||
/**
|
||||
* Get file content from project cache or reuse from universal cache if available
|
||||
* This method efficiently searches across all available caches to find file content
|
||||
*/
|
||||
async getOrReuseFileContext(project: ProjectConfig, filePath: string): Promise<string | null> {
|
||||
try {
|
||||
// 1. Try to get from project cache first
|
||||
const projectContent = await this.getFileContext(project, filePath);
|
||||
if (projectContent) {
|
||||
return projectContent;
|
||||
}
|
||||
|
||||
// 2. Search other projects as fallback
|
||||
const result = await this.searchOtherProjectsForFile(filePath);
|
||||
if (result) {
|
||||
// Associate with current project
|
||||
await this.associateCacheWithProject(project, filePath, result.cacheKey);
|
||||
logInfo(
|
||||
`Reused cached content from other project for: ${filePath} in project ${project.name}`
|
||||
);
|
||||
return result.content;
|
||||
}
|
||||
|
||||
// No content found in any cache
|
||||
return null;
|
||||
} catch (error) {
|
||||
logError(`Error in getOrReuseFileContext for ${filePath} in project ${project.name}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get content for a specific file in a project
|
||||
*/
|
||||
async getFileContext(project: ProjectConfig, filePath: string): Promise<string | null> {
|
||||
protected async getFileContext(project: ProjectConfig, filePath: string): Promise<string | null> {
|
||||
try {
|
||||
// Ensure filePath is valid before proceeding
|
||||
if (!filePath || typeof filePath !== "string") {
|
||||
|
|
@ -445,8 +552,7 @@ export class ProjectContextCache {
|
|||
* Add or update a file in a project's context
|
||||
*/
|
||||
async setFileContext(project: ProjectConfig, filePath: string, content: string): Promise<void> {
|
||||
try {
|
||||
const cache = (await this.get(project)) || this.createEmptyCache();
|
||||
return await this.updateCacheSafelyAsync(project, async (cache) => {
|
||||
if (!cache.fileContexts) {
|
||||
cache.fileContexts = {};
|
||||
}
|
||||
|
|
@ -469,46 +575,108 @@ export class ProjectContextCache {
|
|||
cacheKey,
|
||||
};
|
||||
|
||||
await this.set(project, cache);
|
||||
logInfo(`Added/updated file context for ${filePath} in project ${project.name}`);
|
||||
} catch (error) {
|
||||
logError(`Error setting file context for ${filePath}:`, error);
|
||||
}
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a file from a project's context
|
||||
*/
|
||||
async removeFileContext(project: ProjectConfig, filePath: string): Promise<void> {
|
||||
try {
|
||||
const cache = await this.get(project);
|
||||
if (cache && cache.fileContexts[filePath]) {
|
||||
return await this.updateCacheSafelyAsync(project, async (cache) => {
|
||||
if (cache.fileContexts && cache.fileContexts[filePath]) {
|
||||
// Get the cache key before removing from project cache
|
||||
const { cacheKey } = cache.fileContexts[filePath];
|
||||
|
||||
// Remove from project cache
|
||||
delete cache.fileContexts[filePath];
|
||||
await this.set(project, cache);
|
||||
|
||||
// Remove from file cache
|
||||
await this.fileCache.remove(cacheKey);
|
||||
|
||||
logInfo(`Removed file context for ${filePath} in project ${project.name}`);
|
||||
}
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search all existing projects for cached content of a file
|
||||
* If found, migrate that content to universal cache and return it
|
||||
*/
|
||||
private async searchOtherProjectsForFile(
|
||||
filePath: string
|
||||
): Promise<{ cacheKey: string; content: string } | null> {
|
||||
try {
|
||||
const settings = getSettings();
|
||||
const projects = settings.projectList || [];
|
||||
|
||||
if (projects.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
logInfo(`Searching other projects for file: ${filePath}`);
|
||||
|
||||
for (const project of projects) {
|
||||
// Skip projects without cache
|
||||
const cache = await this.get(project);
|
||||
if (!cache || !cache.fileContexts) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this project has the file cached
|
||||
if (cache.fileContexts[filePath]) {
|
||||
const { cacheKey } = cache.fileContexts[filePath];
|
||||
if (!cacheKey) continue;
|
||||
|
||||
// Try to get content from this project's cache
|
||||
const content = await this.fileCache.get(cacheKey);
|
||||
if (content) {
|
||||
logInfo(`Found content for file ${filePath} in project ${project.name}`);
|
||||
return { content, cacheKey };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logInfo(`No content found in any project for file: ${filePath}`);
|
||||
return null;
|
||||
} catch (error) {
|
||||
logError(`Error removing file context for ${filePath}:`, error);
|
||||
logError(`Error searching other projects for file ${filePath}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update project file references based on inclusion/exclusion patterns.
|
||||
* Removes references to files that no longer match patterns, but keeps their content cached.
|
||||
* Associate an existing cache with a specific project, creates the project reference to that cache
|
||||
*/
|
||||
async cleanupProjectFileReferences(project: ProjectConfig): Promise<void> {
|
||||
try {
|
||||
const cache = await this.get(project);
|
||||
if (!cache || !cache.fileContexts) {
|
||||
return;
|
||||
async associateCacheWithProject(
|
||||
project: ProjectConfig,
|
||||
filePath: string,
|
||||
cacheKey: string
|
||||
): Promise<void> {
|
||||
return await this.updateCacheSafelyAsync(project, async (cache) => {
|
||||
if (!cache.fileContexts) {
|
||||
cache.fileContexts = {};
|
||||
}
|
||||
|
||||
// Update project context to reference the other cache key directly
|
||||
cache.fileContexts[filePath] = {
|
||||
timestamp: Date.now(),
|
||||
cacheKey: cacheKey,
|
||||
};
|
||||
|
||||
logInfo(`Associated cache with project ${project.name} for file: ${filePath}`);
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to perform file reference cleanup logic on a cache object
|
||||
*/
|
||||
private cleanupFileReferencesInCache(project: ProjectConfig, cache: ContextCache): ContextCache {
|
||||
if (!cache.fileContexts) {
|
||||
return cache;
|
||||
}
|
||||
|
||||
const { inclusions, exclusions } = getMatchingPatterns({
|
||||
|
|
@ -537,11 +705,26 @@ export class ProjectContextCache {
|
|||
// Only update if we actually removed something
|
||||
if (removedCount > 0) {
|
||||
cache.fileContexts = updatedFileContexts;
|
||||
await this.set(project, cache);
|
||||
logInfo(
|
||||
`Removed ${removedCount} file references from project ${project.name} that no longer match inclusion patterns`
|
||||
);
|
||||
}
|
||||
|
||||
return cache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update project file references based on inclusion/exclusion patterns.
|
||||
* Removes references to files that no longer match patterns, but keeps their content cached.
|
||||
*/
|
||||
async cleanupProjectFileReferences(project: ProjectConfig): Promise<void> {
|
||||
logInfo(`[cleanupProjectFileReferences] Starting for project: ${project.name}`);
|
||||
try {
|
||||
await this.updateCacheSafely(
|
||||
project,
|
||||
(cache) => this.cleanupFileReferencesInCache(project, cache),
|
||||
true
|
||||
);
|
||||
} catch (error) {
|
||||
logError(`Error cleaning up project file references for ${project.name}:`, error);
|
||||
}
|
||||
|
|
@ -598,33 +781,82 @@ export class ProjectContextCache {
|
|||
return contextCacheToUpdate;
|
||||
}
|
||||
|
||||
updateProjectMarkdownFilesFromPatterns(
|
||||
project: ProjectConfig,
|
||||
contextCacheToUpdate: ContextCache,
|
||||
projectAllFiles: TFile[]
|
||||
): ContextCache {
|
||||
try {
|
||||
if (!contextCacheToUpdate.fileContexts) {
|
||||
contextCacheToUpdate.fileContexts = {};
|
||||
}
|
||||
|
||||
const allFiles = projectAllFiles.filter((file) => file.extension === "md");
|
||||
let addedCount = 0;
|
||||
|
||||
for (const file of allFiles) {
|
||||
if (contextCacheToUpdate.fileContexts[file.path]) {
|
||||
continue;
|
||||
}
|
||||
const cacheKey = this.fileCache.getCacheKey(file, project.id);
|
||||
contextCacheToUpdate.fileContexts[file.path] = {
|
||||
timestamp: Date.now(),
|
||||
cacheKey,
|
||||
};
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
if (addedCount > 0) {
|
||||
logInfo(
|
||||
`[updateProjectFilesFromPatterns] Project ${project.name}: Added ${addedCount} new file references to context (in memory).`
|
||||
);
|
||||
}
|
||||
|
||||
logInfo(
|
||||
`[updateProjectFilesFromPatterns] Completed for project: ${project.name}. Total markdown fileContexts in memory: ${Object.keys(contextCacheToUpdate.fileContexts).length}`
|
||||
);
|
||||
} catch (error) {
|
||||
logError(`[updateProjectFilesFromPatterns] Error for project ${project.name}:`, error);
|
||||
}
|
||||
return contextCacheToUpdate;
|
||||
}
|
||||
|
||||
//===========================================================================
|
||||
// WEB CONTEXT OPERATIONS
|
||||
//===========================================================================
|
||||
|
||||
/**
|
||||
* Remove a web URL from a project's context
|
||||
* Remove a web URLs from a project's context
|
||||
*/
|
||||
async removeWebUrl(project: ProjectConfig, url: string): Promise<void> {
|
||||
const cache = await this.get(project);
|
||||
if (cache?.webContexts?.[url]) {
|
||||
|
||||
async removeWebUrls(project: ProjectConfig, urls: string[]): Promise<void> {
|
||||
if (!urls.length) return;
|
||||
|
||||
await this.updateCacheSafely(project, (cache) => {
|
||||
if (cache.webContexts) {
|
||||
for (const url of urls) {
|
||||
if (cache.webContexts[url]) {
|
||||
delete cache.webContexts[url];
|
||||
await this.set(project, cache);
|
||||
logInfo(`Removed web context for URL ${url} in project ${project.name}`);
|
||||
}
|
||||
}
|
||||
logInfo(`Removed web contexts for URLs ${urls.join(", ")} in project ${project.name}`);
|
||||
}
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add or update a web URL in a project's context
|
||||
*/
|
||||
async updateWebUrl(project: ProjectConfig, url: string, content: string): Promise<void> {
|
||||
const cache = (await this.get(project)) || this.createEmptyCache();
|
||||
return await this.updateCacheSafely(project, (cache) => {
|
||||
if (!cache.webContexts) {
|
||||
cache.webContexts = {};
|
||||
}
|
||||
cache.webContexts[url] = content;
|
||||
await this.set(project, cache);
|
||||
logInfo(`Updated web context for URL ${url} in project ${project.name}`);
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
//===========================================================================
|
||||
|
|
@ -632,27 +864,123 @@ export class ProjectContextCache {
|
|||
//===========================================================================
|
||||
|
||||
/**
|
||||
* Remove a YouTube URL from a project's context
|
||||
* Remove a YouTube URLs from a project's context
|
||||
*/
|
||||
async removeYoutubeUrl(project: ProjectConfig, url: string): Promise<void> {
|
||||
const cache = await this.get(project);
|
||||
if (cache?.youtubeContexts?.[url]) {
|
||||
|
||||
async removeYoutubeUrls(project: ProjectConfig, urls: string[]): Promise<void> {
|
||||
if (!urls.length) return;
|
||||
|
||||
await this.updateCacheSafely(project, (cache) => {
|
||||
if (cache.youtubeContexts) {
|
||||
for (const url of urls) {
|
||||
if (cache.youtubeContexts[url]) {
|
||||
delete cache.youtubeContexts[url];
|
||||
await this.set(project, cache);
|
||||
logInfo(`Removed YouTube context for URL ${url} in project ${project.name}`);
|
||||
}
|
||||
}
|
||||
logInfo(
|
||||
`removeYoutubeUrls: Removed YouTube contexts for URLs ${urls.join(", ")} in project ${project.name}`
|
||||
);
|
||||
}
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add or update a YouTube URL in a project's context
|
||||
*/
|
||||
async updateYoutubeUrl(project: ProjectConfig, url: string, content: string): Promise<void> {
|
||||
const cache = (await this.get(project)) || this.createEmptyCache();
|
||||
return await this.updateCacheSafely(project, (cache) => {
|
||||
if (!cache.youtubeContexts) {
|
||||
cache.youtubeContexts = {};
|
||||
}
|
||||
cache.youtubeContexts[url] = content;
|
||||
await this.set(project, cache);
|
||||
logInfo(`Updated YouTube context for URL ${url} in project ${project.name}`);
|
||||
return cache;
|
||||
});
|
||||
}
|
||||
|
||||
//===========================================================================
|
||||
// EXTERNAL SAFE OPERATIONS
|
||||
//===========================================================================
|
||||
|
||||
/**
|
||||
* Safe external method for bulk cache updates
|
||||
* This is for external modules that need to perform complex updates safely
|
||||
* @param project
|
||||
* @param updateFn
|
||||
* @param skipIfEmpty - If true, skip the update when cache is empty instead of throwing an error
|
||||
*/
|
||||
async updateCacheSafely(
|
||||
project: ProjectConfig,
|
||||
updateFn: (cache: ContextCache) => ContextCache,
|
||||
skipIfEmpty: boolean = false
|
||||
): Promise<void> {
|
||||
const mutex = await this.getOrCreateProjectMutex(project);
|
||||
|
||||
return await mutex.runExclusive(async () => {
|
||||
try {
|
||||
const cache = await this.get(project);
|
||||
if (!cache) {
|
||||
if (skipIfEmpty) {
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`Project: ${project.name} context cache not found, please invoke getOrInitializeCache method before invoke update context cache`
|
||||
);
|
||||
}
|
||||
const updatedCache = updateFn(cache);
|
||||
await this.setWithoutMutex(project, updatedCache);
|
||||
} catch (error) {
|
||||
logError(`Error updating cache for project ${project.name}:`, error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Safe external method for async cache updates
|
||||
* This is for external modules that need to perform async updates safely
|
||||
* @param project
|
||||
* @param updateFn
|
||||
* @param skipIfEmpty - If true, skip the update when cache is empty instead of throwing an error
|
||||
*/
|
||||
async updateCacheSafelyAsync(
|
||||
project: ProjectConfig,
|
||||
updateFn: (cache: ContextCache) => Promise<ContextCache>,
|
||||
skipIfEmpty: boolean = false
|
||||
): Promise<void> {
|
||||
const mutex = await this.getOrCreateProjectMutex(project);
|
||||
|
||||
return await mutex.runExclusive(async () => {
|
||||
try {
|
||||
const cache = await this.get(project);
|
||||
if (!cache) {
|
||||
if (skipIfEmpty) {
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`Project: ${project.name} context cache not found, please invoke getOrInitializeCache method before invoke update context cache`
|
||||
);
|
||||
}
|
||||
const updatedCache = await updateFn(cache);
|
||||
await this.setWithoutMutex(project, updatedCache);
|
||||
} catch (error) {
|
||||
logError(`Error updating cache for project ${project.name}:`, error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Safe external method for setting complete cache
|
||||
* Use this instead of direct set() calls from external modules
|
||||
*/
|
||||
async setCacheSafely(project: ProjectConfig, contextCache: ContextCache): Promise<void> {
|
||||
const mutex = await this.getOrCreateProjectMutex(project);
|
||||
|
||||
return await mutex.runExclusive(async () => {
|
||||
logInfo(`External safe set for project: ${project.name}`);
|
||||
return await this.setWithoutMutex(project, contextCache);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import {
|
|||
useModelKey,
|
||||
useSelectedTextContexts,
|
||||
} from "@/aiParams";
|
||||
import { useProjectContextStatus } from "@/hooks/useProjectContextStatus";
|
||||
import { ChainType } from "@/chainFactory";
|
||||
|
||||
import { ChatControls, reloadCurrentProject } from "@/components/chat-components/ChatControls";
|
||||
|
|
@ -30,6 +31,7 @@ import { err2String } from "@/utils";
|
|||
import { Buffer } from "buffer";
|
||||
import { Notice, TFile } from "obsidian";
|
||||
import React, { useCallback, useContext, useEffect, useRef, useState } from "react";
|
||||
import ProgressCard from "@/components/project/progress-card";
|
||||
|
||||
type ChatMode = "default" | "project";
|
||||
|
||||
|
|
@ -66,7 +68,29 @@ const Chat: React.FC<ChatProps> = ({
|
|||
const [includeActiveNote, setIncludeActiveNote] = useState(false);
|
||||
const [selectedImages, setSelectedImages] = useState<File[]>([]);
|
||||
const [showChatUI, setShowChatUI] = useState(false);
|
||||
// null: keep default behavior; true: show; false: hide
|
||||
const [progressCardVisible, setProgressCardVisible] = useState<boolean | null>(null);
|
||||
|
||||
const [selectedTextContexts] = useSelectedTextContexts();
|
||||
const projectContextStatus = useProjectContextStatus();
|
||||
|
||||
// Calculate whether to show ProgressCard based on status and user preference
|
||||
const shouldShowProgressCard = () => {
|
||||
if (selectedChain !== ChainType.PROJECT_CHAIN) return false;
|
||||
|
||||
// If user has explicitly set visibility, respect that choice
|
||||
if (progressCardVisible !== null) {
|
||||
return progressCardVisible;
|
||||
}
|
||||
|
||||
// Default behavior: show for loading/error, hide for success
|
||||
return projectContextStatus === "loading" || projectContextStatus === "error";
|
||||
};
|
||||
|
||||
// Reset user preference when status changes to allow default behavior
|
||||
useEffect(() => {
|
||||
setProgressCardVisible(null);
|
||||
}, [projectContextStatus]);
|
||||
|
||||
const [previousMode, setPreviousMode] = useState<ChainType | null>(null);
|
||||
const [selectedChain, setSelectedChain] = useChainType();
|
||||
|
|
@ -535,6 +559,17 @@ const Chat: React.FC<ChatProps> = ({
|
|||
onReplaceChat={setInputMessage}
|
||||
showHelperComponents={selectedChain !== ChainType.PROJECT_CHAIN}
|
||||
/>
|
||||
{shouldShowProgressCard() ? (
|
||||
<div className="tw-inset-0 tw-z-modal tw-flex tw-items-center tw-justify-center tw-rounded-xl">
|
||||
<ProgressCard
|
||||
plugin={plugin}
|
||||
setHiddenCard={() => {
|
||||
setProgressCardVisible(false);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<ChatControls
|
||||
onNewChat={handleNewChat}
|
||||
onSaveAsNote={() => handleSaveAsNote()}
|
||||
|
|
@ -566,7 +601,12 @@ const Chat: React.FC<ChatProps> = ({
|
|||
disableModelSwitch={selectedChain === ChainType.PROJECT_CHAIN}
|
||||
selectedTextContexts={selectedTextContexts}
|
||||
onRemoveSelectedText={handleRemoveSelectedText}
|
||||
showProgressCard={() => {
|
||||
setProgressCardVisible(true);
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
|
|
@ -597,6 +637,9 @@ const Chat: React.FC<ChatProps> = ({
|
|||
}
|
||||
}}
|
||||
showChatUI={(v) => setShowChatUI(v)}
|
||||
onProjectClose={() => {
|
||||
setProgressCardVisible(null);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
import { Plus, X } from "lucide-react";
|
||||
import { AlertCircle, CheckCircle, CircleDashed, Loader2, Plus, X } from "lucide-react";
|
||||
import { TFile } from "obsidian";
|
||||
import React from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { SelectedTextContext } from "@/types/message";
|
||||
import { ChainType } from "@/chainFactory";
|
||||
import { Separator } from "@/components/ui/separator";
|
||||
import { useChainType } from "@/aiParams";
|
||||
import { useProjectContextStatus } from "@/hooks/useProjectContextStatus";
|
||||
|
||||
interface ChatContextMenuProps {
|
||||
activeNote: TFile | null;
|
||||
|
|
@ -14,6 +18,7 @@ interface ChatContextMenuProps {
|
|||
onRemoveContext: (path: string) => void;
|
||||
onRemoveUrl: (url: string) => void;
|
||||
onRemoveSelectedText?: (id: string) => void;
|
||||
showProgressCard: () => void;
|
||||
}
|
||||
|
||||
function ContextNote({
|
||||
|
|
@ -102,7 +107,11 @@ export const ChatContextMenu: React.FC<ChatContextMenuProps> = ({
|
|||
onRemoveContext,
|
||||
onRemoveUrl,
|
||||
onRemoveSelectedText,
|
||||
showProgressCard,
|
||||
}) => {
|
||||
const [currentChain] = useChainType();
|
||||
const contextStatus = useProjectContextStatus();
|
||||
|
||||
const uniqueNotes = React.useMemo(() => {
|
||||
const notesMap = new Map(contextNotes.map((note) => [note.path, note]));
|
||||
|
||||
|
|
@ -125,6 +134,20 @@ export const ChatContextMenu: React.FC<ChatContextMenuProps> = ({
|
|||
selectedTextContexts.length > 0 ||
|
||||
!!activeNote;
|
||||
|
||||
// Get contextStatus from the shared hook
|
||||
const getContextStatusIcon = () => {
|
||||
switch (contextStatus) {
|
||||
case "success":
|
||||
return <CheckCircle className="tw-size-4 tw-text-success" />;
|
||||
case "loading":
|
||||
return <Loader2 className="tw-size-4 tw-animate-spin tw-text-loading" />;
|
||||
case "error":
|
||||
return <AlertCircle className="tw-size-4 tw-text-error" />;
|
||||
case "initial":
|
||||
return <CircleDashed className="tw-size-4 tw-text-faint" />;
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="tw-flex tw-w-full tw-items-center tw-gap-1">
|
||||
<div className="tw-flex tw-h-full tw-items-start">
|
||||
|
|
@ -166,6 +189,22 @@ export const ChatContextMenu: React.FC<ChatContextMenuProps> = ({
|
|||
/>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{currentChain === ChainType.PROJECT_CHAIN && (
|
||||
<>
|
||||
<Separator orientation="vertical" />
|
||||
<div className="">
|
||||
<Button
|
||||
variant="ghost2"
|
||||
size="fit"
|
||||
className="tw-text-muted"
|
||||
onClick={() => showProgressCard()}
|
||||
>
|
||||
{getContextStatusIcon()}
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import {
|
||||
ProjectConfig,
|
||||
getCurrentProject,
|
||||
ProjectConfig,
|
||||
subscribeToProjectChange,
|
||||
useChainType,
|
||||
useModelKey,
|
||||
|
|
@ -71,6 +71,7 @@ interface ChatInputProps {
|
|||
disableModelSwitch?: boolean;
|
||||
selectedTextContexts?: SelectedTextContext[];
|
||||
onRemoveSelectedText?: (id: string) => void;
|
||||
showProgressCard: () => void;
|
||||
}
|
||||
|
||||
const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>(
|
||||
|
|
@ -93,6 +94,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>(
|
|||
disableModelSwitch,
|
||||
selectedTextContexts,
|
||||
onRemoveSelectedText,
|
||||
showProgressCard,
|
||||
},
|
||||
ref
|
||||
) => {
|
||||
|
|
@ -509,6 +511,7 @@ const ChatInput = forwardRef<{ focus: () => void }, ChatInputProps>(
|
|||
onRemoveUrl={(url: string) => setContextUrls((prev) => prev.filter((u) => u !== url))}
|
||||
selectedTextContexts={selectedTextContexts}
|
||||
onRemoveSelectedText={onRemoveSelectedText}
|
||||
showProgressCard={showProgressCard}
|
||||
/>
|
||||
|
||||
{selectedImages.length > 0 && (
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ interface ChatControlsProps {
|
|||
onRemoveUrl: (url: string) => void;
|
||||
selectedTextContexts?: SelectedTextContext[];
|
||||
onRemoveSelectedText?: (id: string) => void;
|
||||
showProgressCard: () => void;
|
||||
}
|
||||
|
||||
const ContextControl: React.FC<ChatControlsProps> = ({
|
||||
|
|
@ -34,6 +35,7 @@ const ContextControl: React.FC<ChatControlsProps> = ({
|
|||
onRemoveUrl,
|
||||
selectedTextContexts,
|
||||
onRemoveSelectedText,
|
||||
showProgressCard,
|
||||
}) => {
|
||||
const [selectedChain] = useChainType();
|
||||
|
||||
|
|
@ -86,6 +88,7 @@ const ContextControl: React.FC<ChatControlsProps> = ({
|
|||
onRemoveUrl={onRemoveUrl}
|
||||
selectedTextContexts={selectedTextContexts}
|
||||
onRemoveSelectedText={onRemoveSelectedText}
|
||||
showProgressCard={showProgressCard}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -131,6 +131,7 @@ export const ProjectList = memo(
|
|||
showChatUI,
|
||||
onClose,
|
||||
inputRef,
|
||||
onProjectClose,
|
||||
}: {
|
||||
className?: string;
|
||||
projects: ProjectConfig[];
|
||||
|
|
@ -142,6 +143,7 @@ export const ProjectList = memo(
|
|||
showChatUI: (v: boolean) => void;
|
||||
onClose: () => void;
|
||||
inputRef: React.RefObject<HTMLTextAreaElement>;
|
||||
onProjectClose: () => void;
|
||||
}): React.ReactElement => {
|
||||
const [isOpen, setIsOpen] = useState(defaultOpen);
|
||||
const [showChatInput, setShowChatInput] = useState(false);
|
||||
|
|
@ -285,6 +287,7 @@ export const ProjectList = memo(
|
|||
size="icon"
|
||||
onClick={() => {
|
||||
enableOrDisableProject(false);
|
||||
onProjectClose();
|
||||
}}
|
||||
aria-label="Close Current Project"
|
||||
>
|
||||
|
|
|
|||
218
src/components/project/progress-card.tsx
Normal file
218
src/components/project/progress-card.tsx
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
import * as React from "react";
|
||||
import { useState } from "react";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import {
|
||||
AlertCircle,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
FileText,
|
||||
Loader2,
|
||||
RotateCcw,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { FailedItem, useProjectContextLoad } from "@/aiParams";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { TruncatedText } from "@/components/TruncatedText";
|
||||
import CopilotPlugin from "@/main";
|
||||
import { logError } from "@/logger";
|
||||
|
||||
interface ProgressCardProps {
|
||||
plugin?: CopilotPlugin;
|
||||
setHiddenCard: (hidden: boolean) => void;
|
||||
}
|
||||
|
||||
export default function ProgressCard({ plugin, setHiddenCard }: ProgressCardProps) {
|
||||
const [contextLoadState] = useProjectContextLoad();
|
||||
const totalFiles = contextLoadState.total;
|
||||
const successFiles = contextLoadState.success;
|
||||
const failedFiles = contextLoadState.failed;
|
||||
const processingFiles = contextLoadState.processingFiles;
|
||||
|
||||
// Control file list expand/collapse state
|
||||
const [isProcessingExpanded, setIsProcessingExpanded] = useState(false);
|
||||
const [isFailedExpanded, setIsFailedExpanded] = useState(false);
|
||||
|
||||
const processedFilesLen = successFiles.length + failedFiles.length;
|
||||
const progressPercentage =
|
||||
totalFiles.length > 0 ? Math.round((processedFilesLen / totalFiles.length) * 100) : 0;
|
||||
|
||||
const getFailedItemDisplayName = (item: FailedItem): string => {
|
||||
return item.path;
|
||||
};
|
||||
|
||||
// TODO(emt-lin): maybe use it in the future
|
||||
/*const handleRetryAllFailed = () => {
|
||||
console.log("Retrying all failed items");
|
||||
};*/
|
||||
|
||||
const handleRetryFailedItem = async (item: FailedItem) => {
|
||||
if (!plugin?.projectManager) {
|
||||
logError("ProjectManager not available");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await plugin.projectManager.retryFailedItem(item);
|
||||
} catch (error) {
|
||||
logError(`Error retrying failed item: ${error}`);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Card className="tw-w-full tw-border tw-border-solid tw-border-border tw-bg-transparent tw-shadow-none">
|
||||
<CardHeader>
|
||||
<CardTitle className="tw-flex tw-items-center tw-justify-between tw-gap-2">
|
||||
<div className="tw-flex tw-items-center tw-gap-2">
|
||||
<FileText className="tw-size-5" />
|
||||
Context Loading
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost2"
|
||||
className="tw-size-6 tw-p-0 tw-text-muted"
|
||||
title="Close Progress Bar"
|
||||
onClick={() => setHiddenCard(true)}
|
||||
>
|
||||
<X className="tw-size-4" />
|
||||
</Button>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="tw-space-y-6">
|
||||
{/* Total progress display */}
|
||||
<div className="tw-space-y-2">
|
||||
<div className="tw-flex tw-items-center tw-justify-between tw-text-sm">
|
||||
<div className="tw-flex tw-items-center tw-gap-2">
|
||||
<span className="tw-text-muted">Total progress</span>
|
||||
<span className="tw-text-xs tw-text-muted">
|
||||
(Success:{" "}
|
||||
<span className="tw-font-medium tw-text-success">{successFiles.length}</span>,
|
||||
Failed: <span className="tw-font-medium tw-text-error">{failedFiles.length}</span>)
|
||||
</span>
|
||||
</div>
|
||||
<span className="tw-font-medium">
|
||||
{processedFilesLen}/{totalFiles.length} ({progressPercentage}%)
|
||||
</span>
|
||||
</div>
|
||||
<Progress value={progressPercentage} className="tw-h-2" />
|
||||
</div>
|
||||
|
||||
{/* Currently processing file */}
|
||||
{processingFiles.length > 0 && (
|
||||
<div className="tw-space-y-3">
|
||||
<div
|
||||
className="tw--m-1 tw-flex tw-cursor-pointer tw-items-center tw-gap-2 tw-rounded-md tw-p-1 tw-transition-colors hover:tw-bg-muted/10"
|
||||
onClick={() => setIsProcessingExpanded(!isProcessingExpanded)}
|
||||
>
|
||||
<Loader2 className="tw-size-4 tw-animate-spin tw-text-accent" />
|
||||
<span className="tw-text-sm tw-font-medium">Processing</span>
|
||||
{isProcessingExpanded ? (
|
||||
<ChevronDown className="tw-ml-auto tw-size-4" />
|
||||
) : (
|
||||
<ChevronRight className="tw-ml-auto tw-size-4" />
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isProcessingExpanded && (
|
||||
<div className="tw-max-h-32 tw-space-y-2 tw-overflow-y-auto">
|
||||
{processingFiles.map((fileName, index) => (
|
||||
<div
|
||||
key={index}
|
||||
className="tw-flex tw-items-center tw-gap-2 tw-rounded-md tw-p-2 tw-text-sm tw-bg-faint/10"
|
||||
>
|
||||
<div className="tw-size-2 tw-animate-pulse tw-rounded-full tw-bg-interactive-accent" />
|
||||
<TruncatedText className="tw-flex-1" title={fileName}>
|
||||
{fileName}
|
||||
</TruncatedText>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Failed to process the file */}
|
||||
{failedFiles.length > 0 && (
|
||||
<div className="tw-space-y-3">
|
||||
<div className="tw-flex tw-items-center tw-gap-2">
|
||||
<div
|
||||
className="-tw-m-1 tw-flex tw-flex-1 tw-cursor-pointer tw-items-center tw-gap-2 tw-rounded-md tw-p-1 tw-transition-colors hover:tw-bg-muted/10"
|
||||
onClick={() => setIsFailedExpanded(!isFailedExpanded)}
|
||||
>
|
||||
<AlertCircle className="tw-size-4 tw-text-error" />
|
||||
<span className="tw-text-sm tw-font-medium">Failed</span>
|
||||
<Badge variant="destructive" className="tw-text-xs">
|
||||
{failedFiles.length} files
|
||||
</Badge>
|
||||
{isFailedExpanded ? (
|
||||
<ChevronDown className="tw-ml-auto tw-size-4" />
|
||||
) : (
|
||||
<ChevronRight className="tw-ml-auto tw-size-4" />
|
||||
)}
|
||||
</div>
|
||||
{/*todo(emt-lin): in the future, we can add all failed files to retry*/}
|
||||
{/*<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="tw-size-6 tw-bg-transparent tw-p-0"
|
||||
title="Retry failed files"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
// Handle retry logic
|
||||
handleRetryAllFailed()
|
||||
}}
|
||||
>
|
||||
<RotateCcw className="tw-size-3" />
|
||||
</Button>*/}
|
||||
</div>
|
||||
|
||||
{isFailedExpanded && (
|
||||
<div className="tw-max-h-32 tw-space-y-2 tw-overflow-y-auto">
|
||||
{failedFiles.map((failedItem: FailedItem, index: number) => (
|
||||
<div
|
||||
key={index}
|
||||
className="tw-flex tw-items-center tw-gap-2 tw-rounded-md tw-p-2 tw-text-sm tw-bg-faint/10"
|
||||
>
|
||||
<div className="tw-flex tw-min-w-0 tw-flex-1 tw-flex-col tw-gap-1">
|
||||
<div className="tw-flex tw-items-center tw-gap-2">
|
||||
<div className="tw-size-2 tw-rounded-full tw-bg-error/80" />
|
||||
<TruncatedText className="tw-flex-1 tw-font-bold" title={failedItem.path}>
|
||||
{getFailedItemDisplayName(failedItem)}
|
||||
</TruncatedText>
|
||||
</div>
|
||||
<div className="tw-flex tw-items-center tw-gap-2">
|
||||
<div className="tw-size-2 tw-rounded-full" />
|
||||
{failedItem.error && (
|
||||
<TruncatedText
|
||||
className="tw-flex-1 tw-text-xs tw-text-error/80"
|
||||
title={failedItem.error}
|
||||
>
|
||||
<span className="tw-text-sm tw-text-error">Loading Error: </span>
|
||||
{failedItem.error}
|
||||
</TruncatedText>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="tw-size-5 tw-p-0"
|
||||
title={`Retry ${failedItem.type} item`}
|
||||
onClick={async (e) => {
|
||||
e.stopPropagation();
|
||||
await handleRetryFailedItem(failedItem);
|
||||
}}
|
||||
>
|
||||
<RotateCcw className="tw-size-3" />
|
||||
</Button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
26
src/components/ui/progress.tsx
Normal file
26
src/components/ui/progress.tsx
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import * as React from "react";
|
||||
import * as ProgressPrimitive from "@radix-ui/react-progress";
|
||||
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
const Progress = React.forwardRef<
|
||||
React.ElementRef<typeof ProgressPrimitive.Root>,
|
||||
React.ComponentPropsWithoutRef<typeof ProgressPrimitive.Root>
|
||||
>(({ className, value, ...props }, ref) => (
|
||||
<ProgressPrimitive.Root
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"tw-relative tw-h-2 tw-w-full tw-overflow-hidden tw-rounded-full tw-bg-interactive-accent/20 tw-border-interactive-accent/30",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<ProgressPrimitive.Indicator
|
||||
className="tw-size-full tw-flex-1 tw-bg-interactive-accent tw-transition-all"
|
||||
style={{ transform: `translateX(-${100 - Math.min(Math.max(value || 0, 0), 100)}%)` }}
|
||||
/>
|
||||
</ProgressPrimitive.Root>
|
||||
));
|
||||
Progress.displayName = ProgressPrimitive.Root.displayName;
|
||||
|
||||
export { Progress };
|
||||
50
src/hooks/useProjectContextStatus.ts
Normal file
50
src/hooks/useProjectContextStatus.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import { ChainType } from "@/chainFactory";
|
||||
import { useChainType, useProjectLoading, useProjectContextLoad } from "@/aiParams";
|
||||
|
||||
/**
|
||||
* Hook to calculate the project context status based on project loading state and context load state.
|
||||
* Returns one of: 'initial', 'loading', 'success', 'error'
|
||||
*
|
||||
* Status meanings:
|
||||
* - 'initial': Project context loading has not started yet or not in project mode
|
||||
* - 'loading': Project context is currently being loaded or files are being processed
|
||||
* - 'success': All files have been processed successfully without any errors
|
||||
* - 'error': One or more files have failed to process during the loading operation
|
||||
*/
|
||||
|
||||
export type ProjectContextStatus = "initial" | "loading" | "success" | "error";
|
||||
|
||||
export function useProjectContextStatus(): ProjectContextStatus {
|
||||
const [currentChain] = useChainType();
|
||||
const [isProjectLoading] = useProjectLoading();
|
||||
const [contextLoadState] = useProjectContextLoad();
|
||||
|
||||
const contextStatus = (() => {
|
||||
// Only calculate status for project mode
|
||||
if (currentChain !== ChainType.PROJECT_CHAIN) {
|
||||
return "initial";
|
||||
}
|
||||
|
||||
const { total, success, failed, processingFiles } = contextLoadState;
|
||||
|
||||
// Loading state: when project is loading or files are being processed
|
||||
if (isProjectLoading || processingFiles.length > 0) {
|
||||
return "loading";
|
||||
}
|
||||
|
||||
// Error state: when there are failed files
|
||||
if (failed.length > 0) {
|
||||
return "error";
|
||||
}
|
||||
|
||||
// Success state: when all files have been processed successfully
|
||||
if (total.length > 0 && success.length === total.length) {
|
||||
return "success";
|
||||
}
|
||||
|
||||
// Initial state: when no context loading has started yet
|
||||
return "initial";
|
||||
})();
|
||||
|
||||
return contextStatus;
|
||||
}
|
||||
|
|
@ -1,11 +1,13 @@
|
|||
import { ImageProcessor } from "@/imageProcessing/imageProcessor";
|
||||
import { BrevilabsClient, Url4llmResponse } from "@/LLMProviders/brevilabsClient";
|
||||
import { isYoutubeUrl } from "@/utils";
|
||||
import { err2String, isYoutubeUrl } from "@/utils";
|
||||
import { logError } from "@/logger";
|
||||
|
||||
export interface MentionData {
|
||||
type: string;
|
||||
original: string;
|
||||
processed?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export class Mention {
|
||||
|
|
@ -41,24 +43,30 @@ export class Mention {
|
|||
.filter((url) => !isYoutubeUrl(url));
|
||||
}
|
||||
|
||||
async processUrl(url: string): Promise<Url4llmResponse> {
|
||||
async processUrl(url: string): Promise<Url4llmResponse & { error?: string }> {
|
||||
try {
|
||||
return await this.brevilabsClient.url4llm(url);
|
||||
} catch (error) {
|
||||
console.error(`Error processing URL ${url}:`, error);
|
||||
return { response: url, elapsed_time_ms: 0 };
|
||||
const msg = err2String(error);
|
||||
logError(`Error processing URL ${url}: ${msg}`);
|
||||
return { response: url, elapsed_time_ms: 0, error: msg };
|
||||
}
|
||||
}
|
||||
|
||||
// For non-youtube URLs
|
||||
async processUrls(text: string): Promise<{ urlContext: string; imageUrls: string[] }> {
|
||||
async processUrls(text: string): Promise<{
|
||||
urlContext: string;
|
||||
imageUrls: string[];
|
||||
processedErrorUrls: Record<string, string>;
|
||||
}> {
|
||||
const urls = this.extractUrls(text);
|
||||
let urlContext = "";
|
||||
const imageUrls: string[] = [];
|
||||
const processedErrorUrls: Record<string, string> = {};
|
||||
|
||||
// Return empty string if no URLs to process
|
||||
if (urls.length === 0) {
|
||||
return { urlContext: "", imageUrls: [] };
|
||||
return { urlContext, imageUrls, processedErrorUrls };
|
||||
}
|
||||
|
||||
// Process all URLs concurrently
|
||||
|
|
@ -75,6 +83,7 @@ export class Mention {
|
|||
type: "url",
|
||||
original: url,
|
||||
processed: processed.response,
|
||||
error: processed.error,
|
||||
});
|
||||
}
|
||||
return this.mentions.get(url);
|
||||
|
|
@ -87,9 +96,13 @@ export class Mention {
|
|||
if (urlData?.processed) {
|
||||
urlContext += `\n\n<url_content>\n<url>${urlData.original}</url>\n<content>\n${urlData.processed}\n</content>\n</url_content>`;
|
||||
}
|
||||
|
||||
if (urlData?.error) {
|
||||
processedErrorUrls[urlData.original] = urlData.error;
|
||||
}
|
||||
});
|
||||
|
||||
return { urlContext, imageUrls };
|
||||
return { urlContext, imageUrls, processedErrorUrls };
|
||||
}
|
||||
|
||||
getMentions(): Map<string, MentionData> {
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ export function shouldIndexFile(
|
|||
return false;
|
||||
}
|
||||
|
||||
// Project：Only the included files need to be processed.
|
||||
// Project: Only the included files need to be processed.
|
||||
if (isProject && !inclusions) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
/*
|
||||
Copilot Plugin Css
|
||||
This file is generated by Copilot plugin.
|
||||
*/
|
||||
@import "tailwindcss/base";
|
||||
@import "tailwindcss/components";
|
||||
@import "tailwindcss/utilities";
|
||||
|
|
|
|||
|
|
@ -1,8 +1,25 @@
|
|||
import { ProjectConfig } from "@/aiParams";
|
||||
import { ProjectContextCache } from "@/cache/projectContextCache";
|
||||
import { ContextCache, ProjectContextCache } from "@/cache/projectContextCache";
|
||||
|
||||
// Mock dependencies
|
||||
jest.mock("obsidian");
|
||||
jest.mock("obsidian", () => ({
|
||||
TFile: class MockTFile {
|
||||
path: string;
|
||||
extension: string;
|
||||
basename: string;
|
||||
stat: any;
|
||||
|
||||
constructor(path: string, extension: string, basename: string, stat: any) {
|
||||
this.path = path;
|
||||
this.extension = extension;
|
||||
this.basename = basename;
|
||||
this.stat = stat;
|
||||
}
|
||||
},
|
||||
TAbstractFile: class MockTAbstractFile {},
|
||||
Vault: class MockVault {},
|
||||
}));
|
||||
|
||||
jest.mock("@/logger", () => ({
|
||||
logInfo: jest.fn(),
|
||||
logError: jest.fn(),
|
||||
|
|
@ -94,19 +111,16 @@ describe("ProjectContextCache", () => {
|
|||
let mockProject: ProjectConfig;
|
||||
|
||||
// Mock files
|
||||
const mockMarkdownFile = {
|
||||
path: "test/file.md",
|
||||
extension: "md",
|
||||
basename: "file",
|
||||
stat: { mtime: Date.now(), size: 100 },
|
||||
} as any;
|
||||
const { TFile: MockedTFile } = jest.requireMock("obsidian");
|
||||
const mockMarkdownFile = new MockedTFile("test/file.md", "md", "file", {
|
||||
mtime: Date.now(),
|
||||
size: 100,
|
||||
});
|
||||
|
||||
const mockPdfFile = {
|
||||
path: "test/document.pdf",
|
||||
extension: "pdf",
|
||||
basename: "document",
|
||||
stat: { mtime: Date.now(), size: 200 },
|
||||
} as any;
|
||||
const mockPdfFile = new MockedTFile("test/document.pdf", "pdf", "document", {
|
||||
mtime: Date.now(),
|
||||
size: 200,
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset mocks
|
||||
|
|
@ -123,7 +137,12 @@ describe("ProjectContextCache", () => {
|
|||
return null;
|
||||
});
|
||||
|
||||
// Get actual instance
|
||||
// Reset vault adapter mocks to default behavior
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(false);
|
||||
mockApp.vault.adapter.read.mockResolvedValue("{}");
|
||||
mockApp.vault.adapter.write.mockResolvedValue(undefined);
|
||||
|
||||
// Get actual instance and clear any existing cache
|
||||
projectContextCache = ProjectContextCache.getInstance();
|
||||
|
||||
// Mock project with minimal properties for testing
|
||||
|
|
@ -141,11 +160,14 @@ describe("ProjectContextCache", () => {
|
|||
const filePath = "test/document.pdf";
|
||||
const content = "PDF content";
|
||||
|
||||
// Initialize a default cache for testing
|
||||
await projectContextCache.getOrInitializeCache(mockProject);
|
||||
|
||||
// Store content
|
||||
await projectContextCache.setFileContext(mockProject, filePath, content);
|
||||
|
||||
// Get content
|
||||
const retrievedContent = await projectContextCache.getFileContext(mockProject, filePath);
|
||||
const retrievedContent = await projectContextCache.getOrReuseFileContext(mockProject, filePath);
|
||||
|
||||
// Verify the content was retrieved (note: implementation may return an object instead of string)
|
||||
expect(retrievedContent).toBeDefined();
|
||||
|
|
@ -176,6 +198,9 @@ describe("ProjectContextCache", () => {
|
|||
});
|
||||
|
||||
test("should clean up project file references", async () => {
|
||||
// Initialize a default cache for testing
|
||||
await projectContextCache.getOrInitializeCache(mockProject);
|
||||
|
||||
// First add some context
|
||||
await projectContextCache.setFileContext(mockProject, mockPdfFile.path, "PDF content");
|
||||
|
||||
|
|
@ -187,4 +212,440 @@ describe("ProjectContextCache", () => {
|
|||
const projectCache = await projectContextCache.get(mockProject);
|
||||
expect(projectCache).toBeDefined();
|
||||
});
|
||||
|
||||
test("should update project markdown files from patterns", () => {
|
||||
// Create an empty context cache to update
|
||||
const contextCache = {
|
||||
markdownContext: "",
|
||||
markdownNeedsReload: true,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Create a test file list containing Markdown and non-Markdown files
|
||||
const testFiles = [
|
||||
{
|
||||
path: "test/file1.md",
|
||||
extension: "md",
|
||||
basename: "file1",
|
||||
stat: { mtime: Date.now(), size: 100 },
|
||||
},
|
||||
{
|
||||
path: "test/file2.md",
|
||||
extension: "md",
|
||||
basename: "file2",
|
||||
stat: { mtime: Date.now(), size: 200 },
|
||||
},
|
||||
{
|
||||
path: "test/document.pdf",
|
||||
extension: "pdf",
|
||||
basename: "document",
|
||||
stat: { mtime: Date.now(), size: 300 },
|
||||
},
|
||||
];
|
||||
|
||||
// Call the method, passing only Markdown files
|
||||
const updatedCache = projectContextCache.updateProjectMarkdownFilesFromPatterns(
|
||||
mockProject,
|
||||
contextCache,
|
||||
testFiles as any
|
||||
);
|
||||
|
||||
// Verify that only Markdown files were added to the cache
|
||||
expect(Object.keys(updatedCache.fileContexts).length).toBe(2);
|
||||
expect(updatedCache.fileContexts["test/file1.md"]).toBeDefined();
|
||||
expect(updatedCache.fileContexts["test/file2.md"]).toBeDefined();
|
||||
expect(updatedCache.fileContexts["test/document.pdf"]).toBeUndefined();
|
||||
|
||||
// Verify that each file entry contains the necessary properties
|
||||
Object.values(updatedCache.fileContexts).forEach((entry) => {
|
||||
expect(entry).toHaveProperty("timestamp");
|
||||
expect(entry).toHaveProperty("cacheKey");
|
||||
expect(typeof entry.timestamp).toBe("number");
|
||||
expect(typeof entry.cacheKey).toBe("string");
|
||||
});
|
||||
});
|
||||
|
||||
test("should update and remove web URLs", async () => {
|
||||
// Create initial context cache
|
||||
const initialCache = {
|
||||
markdownContext: "",
|
||||
markdownNeedsReload: true,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Mock vault to return our initial cache
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(true);
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache));
|
||||
|
||||
// Update Web URL
|
||||
const testUrl = "https://example.com/test";
|
||||
const testContent = "Example web content";
|
||||
await projectContextCache.updateWebUrl(mockProject, testUrl, testContent);
|
||||
|
||||
// Verify write was called
|
||||
expect(mockApp.vault.adapter.write).toHaveBeenCalled();
|
||||
|
||||
// Reset mock to return the updated cache
|
||||
const updatedCache = {
|
||||
...initialCache,
|
||||
webContexts: { [testUrl]: testContent },
|
||||
};
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(updatedCache));
|
||||
|
||||
// Add another URL
|
||||
const testUrl2 = "https://example.com/test2";
|
||||
const testContent2 = "Another example";
|
||||
await projectContextCache.updateWebUrl(mockProject, testUrl2, testContent2);
|
||||
|
||||
// Remove URL
|
||||
const urlsToRemove = [testUrl];
|
||||
await projectContextCache.removeWebUrls(mockProject, urlsToRemove);
|
||||
|
||||
// Final cache should only contain the second URL
|
||||
const finalCache = {
|
||||
...initialCache,
|
||||
webContexts: { [testUrl2]: testContent2 },
|
||||
};
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(finalCache));
|
||||
|
||||
// Get cache check results
|
||||
const resultCache = await projectContextCache.get(mockProject);
|
||||
expect(resultCache).toBeDefined();
|
||||
expect(resultCache?.webContexts[testUrl]).toBeUndefined();
|
||||
expect(resultCache?.webContexts[testUrl2]).toBe(testContent2);
|
||||
});
|
||||
|
||||
test("should update and remove YouTube URLs", async () => {
|
||||
// Create initial context cache
|
||||
const initialCache = {
|
||||
markdownContext: "",
|
||||
markdownNeedsReload: true,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Mock vault to return our initial cache
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(true);
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache));
|
||||
|
||||
// Update YouTube URL
|
||||
const testYoutubeUrl = "https://youtube.com/watch?v=test123";
|
||||
const testYoutubeContent = "Test YouTube transcript";
|
||||
await projectContextCache.updateYoutubeUrl(mockProject, testYoutubeUrl, testYoutubeContent);
|
||||
|
||||
// Verify write was called
|
||||
expect(mockApp.vault.adapter.write).toHaveBeenCalled();
|
||||
|
||||
// Reset mock to return the updated cache
|
||||
const updatedCache = {
|
||||
...initialCache,
|
||||
youtubeContexts: { [testYoutubeUrl]: testYoutubeContent },
|
||||
};
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(updatedCache));
|
||||
|
||||
// Add another URL
|
||||
const testYoutubeUrl2 = "https://youtube.com/watch?v=test456";
|
||||
const testYoutubeContent2 = "Another YouTube transcript";
|
||||
await projectContextCache.updateYoutubeUrl(mockProject, testYoutubeUrl2, testYoutubeContent2);
|
||||
|
||||
// Remove URL
|
||||
const urlsToRemove = [testYoutubeUrl];
|
||||
await projectContextCache.removeYoutubeUrls(mockProject, urlsToRemove);
|
||||
|
||||
// Final cache should only contain the second URL
|
||||
const finalCache = {
|
||||
...initialCache,
|
||||
youtubeContexts: { [testYoutubeUrl2]: testYoutubeContent2 },
|
||||
};
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(finalCache));
|
||||
|
||||
// Get cache check results
|
||||
const resultCache = await projectContextCache.get(mockProject);
|
||||
expect(resultCache).toBeDefined();
|
||||
expect(resultCache?.youtubeContexts[testYoutubeUrl]).toBeUndefined();
|
||||
expect(resultCache?.youtubeContexts[testYoutubeUrl2]).toBe(testYoutubeContent2);
|
||||
});
|
||||
|
||||
test("should safely update cache with updateCacheSafely", async () => {
|
||||
// Create initial context cache
|
||||
const initialCache = {
|
||||
markdownContext: "Initial markdown content",
|
||||
markdownNeedsReload: false,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Mock cache existence
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(true);
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache));
|
||||
|
||||
// Define update function
|
||||
const updateFn = (cache: any) => {
|
||||
cache.markdownContext = "Updated markdown content";
|
||||
cache.markdownNeedsReload = true;
|
||||
return cache;
|
||||
};
|
||||
|
||||
// Execute safe update
|
||||
await projectContextCache.updateCacheSafely(mockProject, updateFn);
|
||||
|
||||
// Verify write was called
|
||||
expect(mockApp.vault.adapter.write).toHaveBeenCalled();
|
||||
|
||||
// Check that written content contains updated values
|
||||
const writeCall = mockApp.vault.adapter.write.mock.calls[0];
|
||||
const writtenContent = JSON.parse(writeCall[1]);
|
||||
expect(writtenContent.markdownContext).toBe("Updated markdown content");
|
||||
expect(writtenContent.markdownNeedsReload).toBe(true);
|
||||
});
|
||||
|
||||
test("should safely update cache with updateCacheSafelyAsync", async () => {
|
||||
// Create initial context cache
|
||||
const initialCache = {
|
||||
markdownContext: "Initial markdown content",
|
||||
markdownNeedsReload: false,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Mock cache existence
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(true);
|
||||
mockApp.vault.adapter.read.mockResolvedValue(JSON.stringify(initialCache));
|
||||
|
||||
// Define async update function
|
||||
const asyncUpdateFn = async (cache: any) => {
|
||||
// Simulate async operation
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
cache.markdownContext = "Async updated content";
|
||||
cache.webContexts = { "https://example.com": "Async web content" };
|
||||
return cache;
|
||||
};
|
||||
|
||||
// Execute async safe update
|
||||
await projectContextCache.updateCacheSafelyAsync(mockProject, asyncUpdateFn);
|
||||
|
||||
// Verify write was called
|
||||
expect(mockApp.vault.adapter.write).toHaveBeenCalled();
|
||||
|
||||
// Check that written content contains updated values
|
||||
const writeCall = mockApp.vault.adapter.write.mock.calls[0];
|
||||
const writtenContent = JSON.parse(writeCall[1]);
|
||||
expect(writtenContent.markdownContext).toBe("Async updated content");
|
||||
expect(writtenContent.webContexts["https://example.com"]).toBe("Async web content");
|
||||
});
|
||||
|
||||
test("should handle skipIfEmpty parameter correctly in updateCacheSafely", async () => {
|
||||
// Create a new project instance to ensure cache isolation
|
||||
const isolatedProject = {
|
||||
id: "isolated-test-project-id",
|
||||
name: "Isolated Test Project",
|
||||
contextSource: {
|
||||
inclusions: "**/*.md, **/*.pdf",
|
||||
exclusions: "",
|
||||
},
|
||||
} as any;
|
||||
|
||||
// Mock cache non-existence for this isolated project
|
||||
mockApp.vault.adapter.exists.mockImplementation((path) => {
|
||||
if (path.includes("isolated-test-project-id")) {
|
||||
return Promise.resolve(false);
|
||||
}
|
||||
return Promise.resolve(true);
|
||||
});
|
||||
|
||||
// Define update function
|
||||
const updateFn = jest.fn((cache: any) => {
|
||||
cache.markdownContext = "Should not be called";
|
||||
return cache;
|
||||
});
|
||||
|
||||
// Execute safe update with skipIfEmpty=true
|
||||
await projectContextCache.updateCacheSafely(isolatedProject, updateFn, true);
|
||||
|
||||
// Verify updateFn was not called
|
||||
expect(updateFn).not.toHaveBeenCalled();
|
||||
|
||||
// Should throw error when skipIfEmpty=false
|
||||
await expect(
|
||||
projectContextCache.updateCacheSafely(isolatedProject, updateFn, false)
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("should handle concurrent updates with updateCacheSafely", async () => {
|
||||
// Create initial context cache with proper types
|
||||
const initialCache: ContextCache = {
|
||||
markdownContext: "Initial markdown content",
|
||||
markdownNeedsReload: false,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Mock cache storage to simulate real persistence
|
||||
let currentCache = initialCache;
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(true);
|
||||
mockApp.vault.adapter.read.mockImplementation(() => {
|
||||
return Promise.resolve(JSON.stringify(currentCache));
|
||||
});
|
||||
mockApp.vault.adapter.write.mockImplementation((path, content) => {
|
||||
currentCache = JSON.parse(content);
|
||||
return Promise.resolve();
|
||||
});
|
||||
|
||||
// Track execution order and update timing
|
||||
const executionOrder: string[] = [];
|
||||
const startTime = Date.now();
|
||||
|
||||
// Create update functions to modify different parts of the cache
|
||||
const updateMarkdown = (cache: any) => {
|
||||
executionOrder.push(`markdown-${Date.now() - startTime}`);
|
||||
cache.markdownContext = "Updated markdown content";
|
||||
return cache;
|
||||
};
|
||||
|
||||
const updateWeb = (cache: any) => {
|
||||
executionOrder.push(`web-${Date.now() - startTime}`);
|
||||
cache.webContexts = { "https://example.com": "Web content" };
|
||||
return cache;
|
||||
};
|
||||
|
||||
const updateYoutube = (cache: any) => {
|
||||
executionOrder.push(`youtube-${Date.now() - startTime}`);
|
||||
cache.youtubeContexts = { "https://youtube.com/test": "YouTube content" };
|
||||
return cache;
|
||||
};
|
||||
|
||||
// Execute concurrent updates
|
||||
await Promise.all([
|
||||
projectContextCache.updateCacheSafely(mockProject, updateMarkdown),
|
||||
projectContextCache.updateCacheSafely(mockProject, updateWeb),
|
||||
projectContextCache.updateCacheSafely(mockProject, updateYoutube),
|
||||
]);
|
||||
|
||||
// Verify write was called three times
|
||||
expect(mockApp.vault.adapter.write).toHaveBeenCalledTimes(3);
|
||||
|
||||
// Verify that all update functions were executed
|
||||
expect(executionOrder.length).toBe(3);
|
||||
|
||||
// Verify all updates were applied to the final cache
|
||||
expect(currentCache.markdownContext).toBe("Updated markdown content");
|
||||
expect(currentCache.webContexts && currentCache.webContexts["https://example.com"]).toBe(
|
||||
"Web content"
|
||||
);
|
||||
expect(
|
||||
currentCache.youtubeContexts && currentCache.youtubeContexts["https://youtube.com/test"]
|
||||
).toBe("YouTube content");
|
||||
});
|
||||
|
||||
test("should handle concurrent updates with updateCacheSafelyAsync", async () => {
|
||||
// Create initial context cache with proper types
|
||||
const initialCache: ContextCache = {
|
||||
markdownContext: "Initial markdown content",
|
||||
markdownNeedsReload: false,
|
||||
webContexts: {},
|
||||
youtubeContexts: {},
|
||||
fileContexts: {},
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
// Mock cache storage to simulate real persistence
|
||||
let currentCache = initialCache;
|
||||
mockApp.vault.adapter.exists.mockResolvedValue(true);
|
||||
mockApp.vault.adapter.read.mockImplementation(() => {
|
||||
return Promise.resolve(JSON.stringify(currentCache));
|
||||
});
|
||||
|
||||
// Track execution order, write calls and timing
|
||||
const executionOrder: string[] = [];
|
||||
const writeOrder: string[] = [];
|
||||
const startTime = Date.now();
|
||||
|
||||
mockApp.vault.adapter.write.mockImplementation((path, content) => {
|
||||
const parsed = JSON.parse(content);
|
||||
currentCache = parsed;
|
||||
|
||||
// Track what was written and when
|
||||
if (parsed.markdownContext === "Async updated markdown") {
|
||||
writeOrder.push(`markdown-${Date.now() - startTime}`);
|
||||
} else if (
|
||||
parsed.webContexts &&
|
||||
parsed.webContexts["https://example.com"] === "Async web content"
|
||||
) {
|
||||
writeOrder.push(`web-${Date.now() - startTime}`);
|
||||
} else if (
|
||||
parsed.youtubeContexts &&
|
||||
parsed.youtubeContexts["https://youtube.com/test"] === "Async YouTube content"
|
||||
) {
|
||||
writeOrder.push(`youtube-${Date.now() - startTime}`);
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
});
|
||||
|
||||
// Create async update functions with different delays
|
||||
const updateMarkdownAsync = async (cache: any) => {
|
||||
const startMark = Date.now() - startTime;
|
||||
executionOrder.push(`markdown-start-${startMark}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
cache.markdownContext = "Async updated markdown";
|
||||
executionOrder.push(`markdown-end-${Date.now() - startTime}`);
|
||||
return cache;
|
||||
};
|
||||
|
||||
const updateWebAsync = async (cache: any) => {
|
||||
const startMark = Date.now() - startTime;
|
||||
executionOrder.push(`web-start-${startMark}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
cache.webContexts = { "https://example.com": "Async web content" };
|
||||
executionOrder.push(`web-end-${Date.now() - startTime}`);
|
||||
return cache;
|
||||
};
|
||||
|
||||
const updateYoutubeAsync = async (cache: any) => {
|
||||
const startMark = Date.now() - startTime;
|
||||
executionOrder.push(`youtube-start-${startMark}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
cache.youtubeContexts = { "https://youtube.com/test": "Async YouTube content" };
|
||||
executionOrder.push(`youtube-end-${Date.now() - startTime}`);
|
||||
return cache;
|
||||
};
|
||||
|
||||
// Execute concurrent async updates
|
||||
await Promise.all([
|
||||
projectContextCache.updateCacheSafelyAsync(mockProject, updateMarkdownAsync),
|
||||
projectContextCache.updateCacheSafelyAsync(mockProject, updateWebAsync),
|
||||
projectContextCache.updateCacheSafelyAsync(mockProject, updateYoutubeAsync),
|
||||
]);
|
||||
|
||||
// Verify all writes occurred
|
||||
expect(mockApp.vault.adapter.write).toHaveBeenCalledTimes(3);
|
||||
expect(writeOrder.length).toBe(3);
|
||||
|
||||
// Verify all updates were applied to the final cache
|
||||
expect(currentCache.markdownContext).toBe("Async updated markdown");
|
||||
expect(currentCache.webContexts && currentCache.webContexts["https://example.com"]).toBe(
|
||||
"Async web content"
|
||||
);
|
||||
expect(
|
||||
currentCache.youtubeContexts && currentCache.youtubeContexts["https://youtube.com/test"]
|
||||
).toBe("Async YouTube content");
|
||||
|
||||
// Log order details for debugging
|
||||
// console.log("Execution order:", executionOrder);
|
||||
// console.log("Write order:", writeOrder);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ export class Docs4LLMParser implements FileParser {
|
|||
throw new Error("No project context provided for file parsing");
|
||||
}
|
||||
|
||||
const cachedContent = await this.projectContextCache.getFileContext(
|
||||
const cachedContent = await this.projectContextCache.getOrReuseFileContext(
|
||||
this.currentProject,
|
||||
file.path
|
||||
);
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ module.exports = {
|
|||
"on-accent-inverted": "var(--text-on-accent-inverted)",
|
||||
success: "var(--text-success)",
|
||||
warning: "var(--text-warning)",
|
||||
loading: "var(--color-blue)",
|
||||
error: "var(--text-error)",
|
||||
accent: "var(--text-accent)",
|
||||
"accent-hover": "var(--text-accent-hover)",
|
||||
|
|
|
|||
Loading…
Reference in a new issue