diff --git a/src/main.ts b/src/main.ts index a4dd38a..8eceb3b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -83,7 +83,8 @@ export default class OpenAugiPlugin extends Plugin { ); this.distillService = new DistillService( this.app, - this.openAIService + this.openAIService, + this.settings ); } @@ -170,14 +171,36 @@ export default class OpenAugiPlugin extends Plugin { // Update services with latest API key this.openAIService = new OpenAIService(this.settings.apiKey); - this.distillService = new DistillService(this.app, this.openAIService); + this.distillService = new DistillService( + this.app, + this.openAIService, + this.settings + ); // Get root content for initial notice const rootContent = await this.app.vault.read(rootFile); - new Notice(`Processing note: ${rootFile.basename}\nCharacters: ${rootContent.length}\nEst. Tokens: ${estimateTokens(rootContent)}`); // Get linked files - const linkedFiles = await this.distillService.getLinkedNotes(rootFile); + let linkedFiles = await this.distillService.getLinkedNotes(rootFile); + + // Deduplicate the linked files by path + const uniqueFiles = new Map(); + for (const file of linkedFiles) { + if (!uniqueFiles.has(file.path)) { + uniqueFiles.set(file.path, file); + } + } + + // Convert back to array + linkedFiles = Array.from(uniqueFiles.values()); + + console.log(`Processing ${linkedFiles.length} unique linked files`); + + if (linkedFiles.length === 0) { + this.loadingIndicator?.hide(); + new Notice('No linked notes found to distill'); + return; + } // Aggregate linked content const { content: linkedContent, sourceNotes } = await this.distillService.aggregateContent(linkedFiles); diff --git a/src/services/distill-service.ts b/src/services/distill-service.ts index 90f7443..e2dd659 100644 --- a/src/services/distill-service.ts +++ b/src/services/distill-service.ts @@ -1,6 +1,7 @@ -import { App, TFile, MetadataCache } from 'obsidian'; +import { App, TFile, MetadataCache, Component } from 'obsidian'; import { OpenAIService } from './openai-service'; import { DistillResponse } from '../types/transcript'; +import { OpenAugiSettings } from '../types/settings'; /** * A simple tokeinzer to estimate the number of tokens @@ -18,10 +19,193 @@ function estimateTokens(text: string): number { export class DistillService { private app: App; private openAIService: OpenAIService; + private settings: OpenAugiSettings; - constructor(app: App, openAIService: OpenAIService) { + constructor(app: App, openAIService: OpenAIService, settings: OpenAugiSettings) { this.app = app; this.openAIService = openAIService; + this.settings = settings; + } + + /** + * Check if a string contains a dataview query + * @param content The content to check + * @returns True if the content contains a dataview query + */ + private containsDataviewQuery(content: string): boolean { + return content.includes("```dataview"); + } + + /** + * Extract dataview queries from content + * @param content The content to extract queries from + * @returns Array of extracted dataview queries + */ + private extractDataviewQueries(content: string): string[] { + const regex = /```dataview\s+([\s\S]*?)```/g; + const queries: string[] = []; + let match; + + while ((match = regex.exec(content)) !== null) { + const query = match[1].trim(); + if (query) { + queries.push(query); + } + } + + return queries; + } + + /** + * Get files from a dataview query + * @param query The dataview query + * @param sourcePath The source file path + * @returns Array of files from dataview query result + */ + private async getFilesFromDataviewQuery(query: string, sourcePath: string): Promise { + // Check if dataview plugin is available + // @ts-ignore - Dataview API is not typed + const dataviewPlugin = this.app.plugins.plugins["dataview"]; + if (!dataviewPlugin?.api) { + console.log("Dataview plugin not available"); + return []; + } + + const dvApi = dataviewPlugin.api; + const files: TFile[] = []; + + try { + // Use dataview API to execute query + const queryResult = await dvApi.queryMarkdown(query, sourcePath); + + if (queryResult.successful) { + // Process based on query type + if (typeof queryResult.value === "object" && queryResult.value !== null && queryResult.value.type === "list") { + // Handle LIST query result as object + for (const item of queryResult.value.values) { + if (item.type === "file" && item.path) { + const file = this.app.vault.getAbstractFileByPath(item.path); + if (file instanceof TFile) { + files.push(file); + } + } + } + } else if (typeof queryResult.value === "object" && queryResult.value !== null && queryResult.value.type === "table") { + // Handle TABLE query result as object + for (const row of queryResult.value.values) { + if (row[0]?.path) { + const file = this.app.vault.getAbstractFileByPath(row[0].path); + if (file instanceof TFile) { + files.push(file); + } + } + } + } else if (typeof queryResult.value === "string") { + // Extract all kinds of links that might appear in dataview output + this.extractLinksFromString(queryResult.value, sourcePath, files); + } + } + } catch (error) { + console.error("Error executing dataview query:", error); + } + + return files; + } + + /** + * Extract links from a string and resolve them to files + * @param content The string content to search for links + * @param sourcePath The source file path for resolving links + * @param files Array to add found files to + */ + private extractLinksFromString(content: string, sourcePath: string, files: TFile[]): void { + // Try multiple patterns to extract links + + // Standard markdown links: [[link]] or [[link|alias]] + const standardLinkRegex = /\[\[(.*?)\]\]/g; + this.processRegexMatches(standardLinkRegex, content, sourcePath, files); + + // Dataview specific format links: [link](link.md) or [alias](link.md) + const markdownLinkRegex = /\[(.*?)\]\((.*?)\)/g; + let match; + while ((match = markdownLinkRegex.exec(content)) !== null) { + // Use the URL part (second group) as the link + const linkPath = match[2]; + this.resolveAndAddFile(linkPath, sourcePath, files); + } + + // Handle line items that might be paths + const lines = content.split("\n"); + for (const line of lines) { + // Check if line is a list item with potential path (starts with - or *) + const trimmedLine = line.trim(); + if ((trimmedLine.startsWith("- ") || trimmedLine.startsWith("* ")) && !trimmedLine.includes("[[") && !trimmedLine.includes("](")) { + // Extract the potential path (remove the list marker and trim) + const potentialPath = trimmedLine.substring(2).trim(); + // If it looks like a path (contains '/' or '.md' or doesn't have spaces) + if (potentialPath.includes("/") || potentialPath.endsWith(".md") || !potentialPath.includes(" ")) { + this.resolveAndAddFile(potentialPath, sourcePath, files); + } + } + } + } + + /** + * Process regex matches to extract links + * @param regex The regex to use + * @param content The content to search + * @param sourcePath The source path for resolving links + * @param files Array to add found files to + */ + private processRegexMatches(regex: RegExp, content: string, sourcePath: string, files: TFile[]): void { + let match; + while ((match = regex.exec(content)) !== null) { + let linkPath = match[1]; + + // Handle aliased links: [[path|alias]] -> path + if (linkPath.includes("|")) { + linkPath = linkPath.split("|")[0]; + } + + this.resolveAndAddFile(linkPath, sourcePath, files); + } + } + + /** + * Resolve a path to a file and add it to the files array if found + * @param linkPath The path to resolve + * @param sourcePath The source path for resolving + * @param files Array to add the file to if found + */ + private resolveAndAddFile(linkPath: string, sourcePath: string, files: TFile[]): void { + // Check if the path already has an extension + const hasExtension = /\.[a-zA-Z0-9]+$/.test(linkPath); + + // Only add .md extension if no extension exists + if (!hasExtension) { + linkPath += ".md"; + } + + // Try to resolve the file using the Obsidian metadata API first + const resolvedFile = this.app.metadataCache.getFirstLinkpathDest(linkPath, sourcePath); + + // Fallback to direct path resolution if needed + let file = null; + if (resolvedFile instanceof TFile) { + file = resolvedFile; + } else if (!resolvedFile) { + const abstractFile = this.app.vault.getAbstractFileByPath(linkPath); + if (abstractFile instanceof TFile) { + file = abstractFile; + } + } + + if (file instanceof TFile) { + // Avoid duplicate files + if (!files.some(existingFile => existingFile.path === file.path)) { + files.push(file); + } + } } /** @@ -30,10 +214,38 @@ export class DistillService { * @returns Array of linked TFiles */ async getLinkedNotes(file: TFile): Promise { - const linkedFiles: TFile[] = []; + let linkedFiles: TFile[] = []; - // Get metadata cache for the current file + // First check if content contains dataview query and settings allow using dataview + if (this.settings.useDataviewIfAvailable) { + const fileContent = await this.app.vault.read(file); + + if (this.containsDataviewQuery(fileContent)) { + const queries = this.extractDataviewQueries(fileContent); + + if (queries.length > 0) { + // Process each query and collect results + for (let i = 0; i < queries.length; i++) { + const query = queries[i]; + + const dataviewFiles = await this.getFilesFromDataviewQuery(query, file.path); + + if (dataviewFiles.length > 0) { + // Add new files that aren't already in the linked files array + for (const dataviewFile of dataviewFiles) { + if (!linkedFiles.some(existingFile => existingFile.path === dataviewFile.path)) { + linkedFiles.push(dataviewFile); + } + } + } + } + } + } + } + + // Always extract regular links as well, and combine with dataview results const metadataCache = this.app.metadataCache.getFileCache(file); + const initialCount = linkedFiles.length; if (metadataCache?.links) { for (const link of metadataCache.links) { @@ -41,7 +253,10 @@ export class DistillService { const linkedFile = this.app.metadataCache.getFirstLinkpathDest(link.link, file.path); if (linkedFile && linkedFile instanceof TFile) { - linkedFiles.push(linkedFile); + // Check if the file is already in the linkedFiles array + if (!linkedFiles.some(existingFile => existingFile.path === linkedFile.path)) { + linkedFiles.push(linkedFile); + } } } } @@ -52,12 +267,26 @@ export class DistillService { const linkedFile = this.app.metadataCache.getFirstLinkpathDest(embed.link, file.path); if (linkedFile && linkedFile instanceof TFile) { - linkedFiles.push(linkedFile); + // Check if the file is already in the linkedFiles array + if (!linkedFiles.some(existingFile => existingFile.path === linkedFile.path)) { + linkedFiles.push(linkedFile); + } } } } - return linkedFiles; + // Final deduplication step + const uniquePaths = new Set(); + const uniqueFiles: TFile[] = []; + + for (const linkedFile of linkedFiles) { + if (!uniquePaths.has(linkedFile.path)) { + uniquePaths.add(linkedFile.path); + uniqueFiles.push(linkedFile); + } + } + + return uniqueFiles; } /** @@ -69,12 +298,26 @@ export class DistillService { let aggregatedContent = ""; const sourceNotes: string[] = []; + // Create a map to track processed files by path to avoid duplicates + const processedFiles = new Map(); + for (const file of files) { + // Skip if we've already processed this file + if (processedFiles.has(file.path)) { + continue; + } + const content = await this.app.vault.read(file); aggregatedContent += `\n\n# Note: ${file.basename}\n\n${content}`; sourceNotes.push(file.basename); + + // Mark this file as processed + processedFiles.set(file.path, true); } + const uniqueFileCount = processedFiles.size; + console.log(`Processed ${uniqueFileCount} unique files`); + return { content: aggregatedContent, sourceNotes }; } @@ -104,11 +347,6 @@ export class DistillService { // Get content from root note const rootContent = await this.app.vault.read(rootFile); - // Log root note content - console.log('Root Note Content:', rootContent); - console.log('Root Note Character Count:', rootContent.length); - console.log('Root Note Estimated Token Count:', estimateTokens(rootContent)); - // Aggregate content from all linked notes const aggregatedResult = await this.aggregateContent(linkedFiles); const linkedContent = aggregatedResult.content; @@ -116,11 +354,6 @@ export class DistillService { // Combine root content with linked content combinedContent = `# Root Note: ${rootFile.basename}\n\n${rootContent}\n\n${linkedContent}`; - - // Log combined content statistics - console.log('Number of Linked Notes:', linkedFiles.length); - console.log('Combined Content Character Count:', combinedContent.length); - console.log('Combined Content Estimated Token Count:', estimateTokens(combinedContent)); } // Send to OpenAI for distillation diff --git a/src/types/settings.ts b/src/types/settings.ts index 202f299..a1995b6 100644 --- a/src/types/settings.ts +++ b/src/types/settings.ts @@ -4,10 +4,12 @@ export interface OpenAugiSettings { apiKey: string; summaryFolder: string; notesFolder: string; + useDataviewIfAvailable: boolean; } export const DEFAULT_SETTINGS: OpenAugiSettings = { apiKey: '', summaryFolder: 'OpenAugi/Summaries', - notesFolder: 'OpenAugi/Notes' + notesFolder: 'OpenAugi/Notes', + useDataviewIfAvailable: true }; \ No newline at end of file diff --git a/src/ui/settings-tab.ts b/src/ui/settings-tab.ts index 5005611..f013c7d 100644 --- a/src/ui/settings-tab.ts +++ b/src/ui/settings-tab.ts @@ -64,5 +64,23 @@ export class OpenAugiSettingTab extends PluginSettingTab { await this.plugin.fileService.ensureDirectoriesExist(); }) ); + + // Check if Dataview plugin is installed + // @ts-ignore - Dataview API is not typed + const dataviewPluginInstalled = this.app.plugins.plugins["dataview"] !== undefined; + + new Setting(containerEl) + .setName('Use Dataview Plugin') + .setDesc(dataviewPluginInstalled + ? 'Process dataview queries in notes to find linked notes' + : 'Dataview plugin is not installed. Install it to enable this feature.') + .addToggle(toggle => toggle + .setValue(this.plugin.settings.useDataviewIfAvailable) + .setDisabled(!dataviewPluginInstalled) + .onChange(async (value) => { + this.plugin.settings.useDataviewIfAvailable = value; + await this.plugin.saveSettings(); + }) + ); } } \ No newline at end of file