working with dataview queries now

src/main.ts

@@ -83,7 +83,8 @@ export default class OpenAugiPlugin extends Plugin {
     );
     this.distillService = new DistillService(
       this.app,
-      this.openAIService
+      this.openAIService,
+      this.settings
     );
   }
 
@@ -170,14 +171,36 @@ export default class OpenAugiPlugin extends Plugin {
 
       // Update services with latest API key
       this.openAIService = new OpenAIService(this.settings.apiKey);
-      this.distillService = new DistillService(this.app, this.openAIService);
+      this.distillService = new DistillService(
+        this.app, 
+        this.openAIService,
+        this.settings
+      );
       
       // Get root content for initial notice
       const rootContent = await this.app.vault.read(rootFile);
-      new Notice(`Processing note: ${rootFile.basename}\nCharacters: ${rootContent.length}\nEst. Tokens: ${estimateTokens(rootContent)}`);
       
       // Get linked files
-      const linkedFiles = await this.distillService.getLinkedNotes(rootFile);
+      let linkedFiles = await this.distillService.getLinkedNotes(rootFile);
+      
+      // Deduplicate the linked files by path
+      const uniqueFiles = new Map<string, TFile>();
+      for (const file of linkedFiles) {
+        if (!uniqueFiles.has(file.path)) {
+          uniqueFiles.set(file.path, file);
+        }
+      }
+      
+      // Convert back to array
+      linkedFiles = Array.from(uniqueFiles.values());
+      
+      console.log(`Processing ${linkedFiles.length} unique linked files`);
+      
+      if (linkedFiles.length === 0) {
+        this.loadingIndicator?.hide();
+        new Notice('No linked notes found to distill');
+        return;
+      }
       
       // Aggregate linked content
       const { content: linkedContent, sourceNotes } = await this.distillService.aggregateContent(linkedFiles);

src/services/distill-service.ts

@@ -1,6 +1,7 @@
-import { App, TFile, MetadataCache } from 'obsidian';
+import { App, TFile, MetadataCache, Component } from 'obsidian';
 import { OpenAIService } from './openai-service';
 import { DistillResponse } from '../types/transcript';
+import { OpenAugiSettings } from '../types/settings';
 
 /**
  * A simple tokeinzer to estimate the number of tokens
@@ -18,10 +19,193 @@ function estimateTokens(text: string): number {
 export class DistillService {
   private app: App;
   private openAIService: OpenAIService;
+  private settings: OpenAugiSettings;
   
-  constructor(app: App, openAIService: OpenAIService) {
+  constructor(app: App, openAIService: OpenAIService, settings: OpenAugiSettings) {
     this.app = app;
     this.openAIService = openAIService;
+    this.settings = settings;
+  }
+
+  /**
+   * Check if a string contains a dataview query
+   * @param content The content to check
+   * @returns True if the content contains a dataview query
+   */
+  private containsDataviewQuery(content: string): boolean {
+    return content.includes("```dataview");
+  }
+
+  /**
+   * Extract dataview queries from content
+   * @param content The content to extract queries from
+   * @returns Array of extracted dataview queries
+   */
+  private extractDataviewQueries(content: string): string[] {
+    const regex = /```dataview\s+([\s\S]*?)```/g;
+    const queries: string[] = [];
+    let match;
+    
+    while ((match = regex.exec(content)) !== null) {
+      const query = match[1].trim();
+      if (query) {
+        queries.push(query);
+      }
+    }
+    
+    return queries;
+  }
+
+  /**
+   * Get files from a dataview query
+   * @param query The dataview query
+   * @param sourcePath The source file path
+   * @returns Array of files from dataview query result
+   */
+  private async getFilesFromDataviewQuery(query: string, sourcePath: string): Promise<TFile[]> {
+    // Check if dataview plugin is available
+    // @ts-ignore - Dataview API is not typed
+    const dataviewPlugin = this.app.plugins.plugins["dataview"];
+    if (!dataviewPlugin?.api) {
+      console.log("Dataview plugin not available");
+      return [];
+    }
+
+    const dvApi = dataviewPlugin.api;
+    const files: TFile[] = [];
+    
+    try {
+      // Use dataview API to execute query
+      const queryResult = await dvApi.queryMarkdown(query, sourcePath);
+
+      if (queryResult.successful) {
+        // Process based on query type
+        if (typeof queryResult.value === "object" && queryResult.value !== null && queryResult.value.type === "list") {
+          // Handle LIST query result as object
+          for (const item of queryResult.value.values) {
+            if (item.type === "file" && item.path) {
+              const file = this.app.vault.getAbstractFileByPath(item.path);
+              if (file instanceof TFile) {
+                files.push(file);
+              }
+            }
+          }
+        } else if (typeof queryResult.value === "object" && queryResult.value !== null && queryResult.value.type === "table") {
+          // Handle TABLE query result as object
+          for (const row of queryResult.value.values) {
+            if (row[0]?.path) {
+              const file = this.app.vault.getAbstractFileByPath(row[0].path);
+              if (file instanceof TFile) {
+                files.push(file);
+              }
+            }
+          }
+        } else if (typeof queryResult.value === "string") {
+          // Extract all kinds of links that might appear in dataview output
+          this.extractLinksFromString(queryResult.value, sourcePath, files);
+        }
+      }
+    } catch (error) {
+      console.error("Error executing dataview query:", error);
+    }
+    
+    return files;
+  }
+  
+  /**
+   * Extract links from a string and resolve them to files
+   * @param content The string content to search for links
+   * @param sourcePath The source file path for resolving links
+   * @param files Array to add found files to
+   */
+  private extractLinksFromString(content: string, sourcePath: string, files: TFile[]): void {
+    // Try multiple patterns to extract links
+    
+    // Standard markdown links: [[link]] or [[link|alias]]
+    const standardLinkRegex = /\[\[(.*?)\]\]/g;
+    this.processRegexMatches(standardLinkRegex, content, sourcePath, files);
+    
+    // Dataview specific format links: [link](link.md) or [alias](link.md)
+    const markdownLinkRegex = /\[(.*?)\]\((.*?)\)/g;
+    let match;
+    while ((match = markdownLinkRegex.exec(content)) !== null) {
+      // Use the URL part (second group) as the link
+      const linkPath = match[2];
+      this.resolveAndAddFile(linkPath, sourcePath, files);
+    }
+    
+    // Handle line items that might be paths
+    const lines = content.split("\n");
+    for (const line of lines) {
+      // Check if line is a list item with potential path (starts with - or *)
+      const trimmedLine = line.trim();
+      if ((trimmedLine.startsWith("- ") || trimmedLine.startsWith("* ")) && !trimmedLine.includes("[[") && !trimmedLine.includes("](")) {
+        // Extract the potential path (remove the list marker and trim)
+        const potentialPath = trimmedLine.substring(2).trim();
+        // If it looks like a path (contains '/' or '.md' or doesn't have spaces)
+        if (potentialPath.includes("/") || potentialPath.endsWith(".md") || !potentialPath.includes(" ")) {
+          this.resolveAndAddFile(potentialPath, sourcePath, files);
+        }
+      }
+    }
+  }
+  
+  /**
+   * Process regex matches to extract links
+   * @param regex The regex to use
+   * @param content The content to search
+   * @param sourcePath The source path for resolving links
+   * @param files Array to add found files to
+   */
+  private processRegexMatches(regex: RegExp, content: string, sourcePath: string, files: TFile[]): void {
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+      let linkPath = match[1];
+      
+      // Handle aliased links: [[path|alias]] -> path
+      if (linkPath.includes("|")) {
+        linkPath = linkPath.split("|")[0];
+      }
+      
+      this.resolveAndAddFile(linkPath, sourcePath, files);
+    }
+  }
+  
+  /**
+   * Resolve a path to a file and add it to the files array if found
+   * @param linkPath The path to resolve
+   * @param sourcePath The source path for resolving
+   * @param files Array to add the file to if found
+   */
+  private resolveAndAddFile(linkPath: string, sourcePath: string, files: TFile[]): void {
+    // Check if the path already has an extension
+    const hasExtension = /\.[a-zA-Z0-9]+$/.test(linkPath);
+    
+    // Only add .md extension if no extension exists
+    if (!hasExtension) {
+      linkPath += ".md";
+    }
+    
+    // Try to resolve the file using the Obsidian metadata API first
+    const resolvedFile = this.app.metadataCache.getFirstLinkpathDest(linkPath, sourcePath);
+    
+    // Fallback to direct path resolution if needed
+    let file = null;
+    if (resolvedFile instanceof TFile) {
+      file = resolvedFile;
+    } else if (!resolvedFile) {
+      const abstractFile = this.app.vault.getAbstractFileByPath(linkPath);
+      if (abstractFile instanceof TFile) {
+        file = abstractFile;
+      }
+    }
+    
+    if (file instanceof TFile) {
+      // Avoid duplicate files
+      if (!files.some(existingFile => existingFile.path === file.path)) {
+        files.push(file);
+      }
+    }
   }
 
   /**
@@ -30,10 +214,38 @@ export class DistillService {
    * @returns Array of linked TFiles
    */
   async getLinkedNotes(file: TFile): Promise<TFile[]> {
-    const linkedFiles: TFile[] = [];
+    let linkedFiles: TFile[] = [];
     
-    // Get metadata cache for the current file
+    // First check if content contains dataview query and settings allow using dataview
+    if (this.settings.useDataviewIfAvailable) {
+      const fileContent = await this.app.vault.read(file);
+      
+      if (this.containsDataviewQuery(fileContent)) {
+        const queries = this.extractDataviewQueries(fileContent);
+        
+        if (queries.length > 0) {
+          // Process each query and collect results
+          for (let i = 0; i < queries.length; i++) {
+            const query = queries[i];
+            
+            const dataviewFiles = await this.getFilesFromDataviewQuery(query, file.path);
+            
+            if (dataviewFiles.length > 0) {
+              // Add new files that aren't already in the linked files array
+              for (const dataviewFile of dataviewFiles) {
+                if (!linkedFiles.some(existingFile => existingFile.path === dataviewFile.path)) {
+                  linkedFiles.push(dataviewFile);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    
+    // Always extract regular links as well, and combine with dataview results
     const metadataCache = this.app.metadataCache.getFileCache(file);
+    const initialCount = linkedFiles.length;
     
     if (metadataCache?.links) {
       for (const link of metadataCache.links) {
@@ -41,7 +253,10 @@ export class DistillService {
         const linkedFile = this.app.metadataCache.getFirstLinkpathDest(link.link, file.path);
         
         if (linkedFile && linkedFile instanceof TFile) {
-          linkedFiles.push(linkedFile);
+          // Check if the file is already in the linkedFiles array
+          if (!linkedFiles.some(existingFile => existingFile.path === linkedFile.path)) {
+            linkedFiles.push(linkedFile);
+          }
         }
       }
     }
@@ -52,12 +267,26 @@ export class DistillService {
         const linkedFile = this.app.metadataCache.getFirstLinkpathDest(embed.link, file.path);
         
         if (linkedFile && linkedFile instanceof TFile) {
-          linkedFiles.push(linkedFile);
+          // Check if the file is already in the linkedFiles array
+          if (!linkedFiles.some(existingFile => existingFile.path === linkedFile.path)) {
+            linkedFiles.push(linkedFile);
+          }
         }
       }
     }
     
-    return linkedFiles;
+    // Final deduplication step
+    const uniquePaths = new Set<string>();
+    const uniqueFiles: TFile[] = [];
+    
+    for (const linkedFile of linkedFiles) {
+      if (!uniquePaths.has(linkedFile.path)) {
+        uniquePaths.add(linkedFile.path);
+        uniqueFiles.push(linkedFile);
+      }
+    }
+    
+    return uniqueFiles;
   }
 
   /**
@@ -69,12 +298,26 @@ export class DistillService {
     let aggregatedContent = "";
     const sourceNotes: string[] = [];
     
+    // Create a map to track processed files by path to avoid duplicates
+    const processedFiles = new Map<string, boolean>();
+    
     for (const file of files) {
+      // Skip if we've already processed this file
+      if (processedFiles.has(file.path)) {
+        continue;
+      }
+      
       const content = await this.app.vault.read(file);
       aggregatedContent += `\n\n# Note: ${file.basename}\n\n${content}`;
       sourceNotes.push(file.basename);
+      
+      // Mark this file as processed
+      processedFiles.set(file.path, true);
     }
     
+    const uniqueFileCount = processedFiles.size;
+    console.log(`Processed ${uniqueFileCount} unique files`);
+    
     return { content: aggregatedContent, sourceNotes };
   }
 
@@ -104,11 +347,6 @@ export class DistillService {
       // Get content from root note
       const rootContent = await this.app.vault.read(rootFile);
       
-      // Log root note content
-      console.log('Root Note Content:', rootContent);
-      console.log('Root Note Character Count:', rootContent.length);
-      console.log('Root Note Estimated Token Count:', estimateTokens(rootContent));
-      
       // Aggregate content from all linked notes
       const aggregatedResult = await this.aggregateContent(linkedFiles);
       const linkedContent = aggregatedResult.content;
@@ -116,11 +354,6 @@ export class DistillService {
       
       // Combine root content with linked content
       combinedContent = `# Root Note: ${rootFile.basename}\n\n${rootContent}\n\n${linkedContent}`;
-      
-      // Log combined content statistics
-      console.log('Number of Linked Notes:', linkedFiles.length);
-      console.log('Combined Content Character Count:', combinedContent.length);
-      console.log('Combined Content Estimated Token Count:', estimateTokens(combinedContent));
     }
     
     // Send to OpenAI for distillation

src/types/settings.ts

@@ -4,10 +4,12 @@ export interface OpenAugiSettings {
   apiKey: string;
   summaryFolder: string;
   notesFolder: string;
+  useDataviewIfAvailable: boolean;
 }
 
 export const DEFAULT_SETTINGS: OpenAugiSettings = {
   apiKey: '',
   summaryFolder: 'OpenAugi/Summaries',
-  notesFolder: 'OpenAugi/Notes'
+  notesFolder: 'OpenAugi/Notes',
+  useDataviewIfAvailable: true
 }; 

src/ui/settings-tab.ts

@@ -64,5 +64,23 @@ export class OpenAugiSettingTab extends PluginSettingTab {
           await this.plugin.fileService.ensureDirectoriesExist();
         })
       );
+      
+    // Check if Dataview plugin is installed
+    // @ts-ignore - Dataview API is not typed
+    const dataviewPluginInstalled = this.app.plugins.plugins["dataview"] !== undefined;
+      
+    new Setting(containerEl)
+      .setName('Use Dataview Plugin')
+      .setDesc(dataviewPluginInstalled 
+        ? 'Process dataview queries in notes to find linked notes'
+        : 'Dataview plugin is not installed. Install it to enable this feature.')
+      .addToggle(toggle => toggle
+        .setValue(this.plugin.settings.useDataviewIfAvailable)
+        .setDisabled(!dataviewPluginInstalled)
+        .onChange(async (value) => {
+          this.plugin.settings.useDataviewIfAvailable = value;
+          await this.plugin.saveSettings();
+        })
+      );
   }
 }