add the distill MOC command

src/main.ts

@@ -1,15 +1,27 @@
-import { Plugin, Notice, TFile, WorkspaceLeaf } from 'obsidian';
+import { Plugin, Notice, TFile } from 'obsidian';
 import { OpenAugiSettings, DEFAULT_SETTINGS } from './types/settings';
 import { OpenAIService } from './services/openai-service';
 import { FileService } from './services/file-service';
+import { DistillService } from './services/distill-service';
 import { OpenAugiSettingTab } from './ui/settings-tab';
 import { LoadingIndicator } from './ui/loading-indicator';
 import { sanitizeFilename } from './utils/filename-utils';
 
+/**
+ * A simple tokeinzer to estimate the number of tokens
+ * @param text Text to count tokens from
+ * @returns Approximate token count
+ */
+function estimateTokens(text: string): number {
+  // Rough estimate: 1 token is approximately 4 characters
+  return Math.ceil(text.length / 4);
+}
+
 export default class OpenAugiPlugin extends Plugin {
   settings: OpenAugiSettings;
   openAIService: OpenAIService;
   fileService: FileService;
+  distillService: DistillService;
   loadingIndicator: LoadingIndicator;
 
   async onload() {
@@ -41,6 +53,20 @@ export default class OpenAugiPlugin extends Plugin {
       }
     });
 
+    // Add command to distill linked notes
+    this.addCommand({
+      id: 'distill-notes',
+      name: 'Distill Linked Notes',
+      callback: async () => {
+        const activeFile = this.app.workspace.getActiveFile();
+        if (activeFile && activeFile.extension === 'md') {
+          await this.distillLinkedNotes(activeFile);
+        } else {
+          new Notice('Please open a markdown file first');
+        }
+      }
+    });
+
     // Add settings tab
     this.addSettingTab(new OpenAugiSettingTab(this.app, this));
   }
@@ -55,6 +81,10 @@ export default class OpenAugiPlugin extends Plugin {
       this.settings.summaryFolder, 
       this.settings.notesFolder
     );
+    this.distillService = new DistillService(
+      this.app,
+      this.openAIService
+    );
   }
 
   /**
@@ -91,6 +121,9 @@ export default class OpenAugiPlugin extends Plugin {
         return;
       }
 
+      // Display character and token count
+      new Notice(`Processing transcript: ${file.basename}\nCharacters: ${content.length}\nEst. Tokens: ${estimateTokens(content)}`);
+
       // Update openAIService with latest API key
       this.openAIService = new OpenAIService(this.settings.apiKey);
       
@@ -104,7 +137,7 @@ export default class OpenAugiPlugin extends Plugin {
       this.loadingIndicator?.hide();
       
       // Show success message
-      new Notice(`Successfully parsed transcript: ${file.basename}`);
+      new Notice(`Successfully parsed transcript: ${file.basename}\nCreated ${parsedData.notes.length} atomic notes`);
       
       // Open the summary file in a new tab
       const sanitizedFilename = sanitizeFilename(file.basename);
@@ -119,6 +152,70 @@ export default class OpenAugiPlugin extends Plugin {
     }
   }
 
+  /**
+   * Distill linked notes
+   * @param rootFile The root file containing links to distill
+   */
+  private async distillLinkedNotes(rootFile: TFile): Promise<void> {
+    try {
+      // Show loading indicator
+      this.loadingIndicator?.show('Distilling linked notes');
+      
+      // Check if API key is set
+      if (!this.settings.apiKey) {
+        this.loadingIndicator?.hide();
+        new Notice('Please set your OpenAI API key in the plugin settings');
+        return;
+      }
+
+      // Update services with latest API key
+      this.openAIService = new OpenAIService(this.settings.apiKey);
+      this.distillService = new DistillService(this.app, this.openAIService);
+      
+      // Get root content for initial notice
+      const rootContent = await this.app.vault.read(rootFile);
+      new Notice(`Processing note: ${rootFile.basename}\nCharacters: ${rootContent.length}\nEst. Tokens: ${estimateTokens(rootContent)}`);
+      
+      // Get linked files
+      const linkedFiles = await this.distillService.getLinkedNotes(rootFile);
+      
+      // Aggregate linked content
+      const { content: linkedContent, sourceNotes } = await this.distillService.aggregateContent(linkedFiles);
+      
+      // Combine content
+      const combinedContent = `# Root Note: ${rootFile.basename}\n\n${rootContent}\n\n${linkedContent}`;
+      const combinedTokens = estimateTokens(combinedContent);
+      
+      // Show combined content notice
+      new Notice(`Combined content from ${linkedFiles.length} linked notes\nTotal characters: ${combinedContent.length}\nEst. total tokens: ${combinedTokens}`);
+      
+      // Distill content from linked notes
+      const distilledData = await this.distillService.distillFromRootNote(
+        rootFile, 
+        combinedContent, 
+        sourceNotes
+      );
+      
+      // Write result to files
+      const summaryPath = await this.fileService.writeDistilledFiles(rootFile, distilledData);
+      
+      // Hide loading indicator
+      this.loadingIndicator?.hide();
+      
+      // Show success message
+      new Notice(`Successfully distilled notes from: ${rootFile.basename}\nCreated ${distilledData.notes.length} atomic notes`);
+      
+      // Open the summary file in a new tab
+      await this.openFileInNewTab(summaryPath);
+    } catch (error) {
+      // Hide loading indicator
+      this.loadingIndicator?.hide();
+      
+      console.error('Failed to distill notes:', error);
+      new Notice('Failed to distill notes. Check console for details.');
+    }
+  }
+
   async loadSettings() {
     this.settings = Object.assign({}, DEFAULT_SETTINGS, await this.loadData());
   }

src/services/distill-service.ts

@@ -0,0 +1,134 @@
+import { App, TFile, MetadataCache } from 'obsidian';
+import { OpenAIService } from './openai-service';
+import { DistillResponse } from '../types/transcript';
+
+/**
+ * A simple tokeinzer to estimate the number of tokens
+ * @param text Text to count tokens from
+ * @returns Approximate token count
+ */
+function estimateTokens(text: string): number {
+  // Rough estimate: 1 token is approximately 4 characters
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Service for distilling content from linked notes
+ */
+export class DistillService {
+  private app: App;
+  private openAIService: OpenAIService;
+  
+  constructor(app: App, openAIService: OpenAIService) {
+    this.app = app;
+    this.openAIService = openAIService;
+  }
+
+  /**
+   * Extract linked notes from a note
+   * @param file The file to extract links from
+   * @returns Array of linked TFiles
+   */
+  async getLinkedNotes(file: TFile): Promise<TFile[]> {
+    const linkedFiles: TFile[] = [];
+    
+    // Get metadata cache for the current file
+    const metadataCache = this.app.metadataCache.getFileCache(file);
+    
+    if (metadataCache?.links) {
+      for (const link of metadataCache.links) {
+        // Get the file for this link
+        const linkedFile = this.app.metadataCache.getFirstLinkpathDest(link.link, file.path);
+        
+        if (linkedFile && linkedFile instanceof TFile) {
+          linkedFiles.push(linkedFile);
+        }
+      }
+    }
+    
+    // Also check for embeds
+    if (metadataCache?.embeds) {
+      for (const embed of metadataCache.embeds) {
+        const linkedFile = this.app.metadataCache.getFirstLinkpathDest(embed.link, file.path);
+        
+        if (linkedFile && linkedFile instanceof TFile) {
+          linkedFiles.push(linkedFile);
+        }
+      }
+    }
+    
+    return linkedFiles;
+  }
+
+  /**
+   * Aggregate content from a set of files
+   * @param files Array of files to aggregate content from
+   * @returns Aggregated content as string along with source file names
+   */
+  async aggregateContent(files: TFile[]): Promise<{content: string, sourceNotes: string[]}> {
+    let aggregatedContent = "";
+    const sourceNotes: string[] = [];
+    
+    for (const file of files) {
+      const content = await this.app.vault.read(file);
+      aggregatedContent += `\n\n# Note: ${file.basename}\n\n${content}`;
+      sourceNotes.push(file.basename);
+    }
+    
+    return { content: aggregatedContent, sourceNotes };
+  }
+
+  /**
+   * Distill content from linked notes in a root note
+   * @param rootFile The root note file to distill from
+   * @param preparedContent Optional preprocessed combined content
+   * @param preparedSourceNotes Optional preprocessed source notes
+   * @returns Distilled content as a DistillResponse
+   */
+  async distillFromRootNote(
+    rootFile: TFile, 
+    preparedContent?: string, 
+    preparedSourceNotes?: string[]
+  ): Promise<DistillResponse> {
+    let combinedContent: string;
+    let sourceNotes: string[];
+    
+    if (preparedContent && preparedSourceNotes) {
+      // Use the provided content and source notes
+      combinedContent = preparedContent;
+      sourceNotes = preparedSourceNotes;
+    } else {
+      // Get all linked notes
+      const linkedFiles = await this.getLinkedNotes(rootFile);
+      
+      // Get content from root note
+      const rootContent = await this.app.vault.read(rootFile);
+      
+      // Log root note content
+      console.log('Root Note Content:', rootContent);
+      console.log('Root Note Character Count:', rootContent.length);
+      console.log('Root Note Estimated Token Count:', estimateTokens(rootContent));
+      
+      // Aggregate content from all linked notes
+      const aggregatedResult = await this.aggregateContent(linkedFiles);
+      const linkedContent = aggregatedResult.content;
+      sourceNotes = aggregatedResult.sourceNotes;
+      
+      // Combine root content with linked content
+      combinedContent = `# Root Note: ${rootFile.basename}\n\n${rootContent}\n\n${linkedContent}`;
+      
+      // Log combined content statistics
+      console.log('Number of Linked Notes:', linkedFiles.length);
+      console.log('Combined Content Character Count:', combinedContent.length);
+      console.log('Combined Content Estimated Token Count:', estimateTokens(combinedContent));
+    }
+    
+    // Send to OpenAI for distillation
+    const distilledContent = await this.openAIService.distillContent(combinedContent);
+    
+    // Add source notes to the response
+    distilledContent.sourceNotes = [rootFile.basename, ...sourceNotes];
+    
+    return distilledContent;
+  }
+} 

src/services/file-service.ts

@@ -1,6 +1,6 @@
 import { App, TFile, Vault } from 'obsidian';
 import { sanitizeFilename, BacklinkMapper } from '../utils/filename-utils';
-import { TranscriptResponse } from '../types/transcript';
+import { TranscriptResponse, DistillResponse } from '../types/transcript';
 
 /**
  * Service for handling file operations
@@ -77,4 +77,58 @@ export class FileService {
       await this.vault.create(`${this.notesFolder}/${sanitizedTitle}.md`, processedContent);
     }
   }
+
+  /**
+   * Write distilled data to files
+   * @param rootFile The root file that was distilled
+   * @param data Distilled data
+   */
+  async writeDistilledFiles(rootFile: TFile, data: DistillResponse): Promise<string> {
+    // Ensure directories exist
+    await this.ensureDirectoriesExist();
+    
+    // Sanitize filename
+    const sanitizedFilename = sanitizeFilename(rootFile.basename);
+    
+    // Register all note titles for backlink processing
+    this.backlinkMapper = new BacklinkMapper(); // Reset the mapper
+    for (const note of data.notes) {
+      const sanitizedTitle = sanitizeFilename(note.title);
+      this.backlinkMapper.registerTitle(note.title, sanitizedTitle);
+    }
+    
+    // Format summary content with source notes and tasks
+    let summaryContent = this.backlinkMapper.processBacklinks(data.summary);
+    
+    // Add source notes section
+    if (data.sourceNotes && data.sourceNotes.length > 0) {
+      const sourceNoteLinks = data.sourceNotes.map(note => 
+        `- [[${note}]]`
+      );
+      summaryContent += '\n\n## Source Notes\n' + sourceNoteLinks.join('\n');
+    }
+    
+    // Add tasks section if there are tasks
+    if (data.tasks && data.tasks.length > 0) {
+      const processedTasks = data.tasks.map(task => 
+        this.backlinkMapper.processBacklinks(task)
+      );
+      summaryContent += '\n\n## Tasks\n' + processedTasks.join('\n');
+    }
+    
+    // Output Summary
+    const summaryPath = `${this.summaryFolder}/${sanitizedFilename} - distilled.md`;
+    await this.vault.create(summaryPath, summaryContent);
+
+    // Output Notes
+    for (const note of data.notes) {
+      // Sanitize note title for filename
+      const sanitizedTitle = sanitizeFilename(note.title);
+      // Process content to ensure backlinks use sanitized filenames
+      const processedContent = this.backlinkMapper.processBacklinks(note.content);
+      await this.vault.create(`${this.notesFolder}/${sanitizedTitle}.md`, processedContent);
+    }
+    
+    return summaryPath;
+  }
 } 

src/services/openai-service.ts

@@ -1,4 +1,14 @@
-import { TranscriptResponse } from '../types/transcript';
+import { TranscriptResponse, DistillResponse } from '../types/transcript';
+
+/**
+ * A simple tokeinzer to estimate the number of tokens
+ * @param text Text to count tokens from
+ * @returns Approximate token count
+ */
+function estimateTokens(text: string): number {
+  // Rough estimate: 1 token is approximately 4 characters
+  return Math.ceil(text.length / 4);
+}
 
 /**
  * Service for handling OpenAI API calls
@@ -79,6 +89,10 @@ ${content}`;
 
     const prompt = this.getPrompt(content);
     
+    // Log prompt statistics
+    console.log('Transcript Prompt Character Count:', prompt.length);
+    console.log('Transcript Prompt Estimated Token Count:', estimateTokens(prompt));
+    
     try {
       const response = await fetch('https://api.openai.com/v1/chat/completions', {
         method: 'POST',
@@ -161,4 +175,139 @@ ${content}`;
       throw error;
     }
   }
+
+  /**
+   * Get the distillation prompt for the OpenAI API
+   * @param content The aggregated content from linked notes
+   * @returns The formatted prompt
+   */
+  private getDistillPrompt(content: string): string {
+    return `
+    You are an expert knowledge curator helping users distill and organize information from their notes. Your task is to analyze multiple related notes and create a coherent set of atomic notes and a summary.
+    
+    # Instructions
+    Analyze the following notes carefully. Consider the title of the note to help you identify distinct concepts, ideas, and insights.
+    The title can be used to help you figure out what's relevant. Some titles might not be helpful in which case you should 
+    determine the intent and most relevant concepts from the content.
+    
+    ### 1. Create Atomic Notes
+    - Identify distinct concepts, ideas, and insights across all notes
+    - Deduplicate and merge overlapping ideas
+    - Any distinct idea should be a separate note
+    - Create self-contained atomic notes with one clear idea per note
+    - Include supporting details and context
+    - Use \`[[Obsidian backlinks]]\` between notes when relevant
+    - Avoid repetition across notes
+    
+    ### 2. Extract Tasks
+    - Identify any actionable tasks present in the notes
+    - Format as: \`- [ ] Description of task [[Linked Atomic Note]]\` (if relevant)
+    - Only include genuinely actionable items, it's okay if there are none
+    
+    ### 3. Create a Summary
+    - Write a concise summary that synthesizes the key concepts
+    - Highlight connections between ideas
+    - Use \`[[Backlinks]]\` to connect to relevant atomic notes
+    
+    # Content to Distill:
+    ${content}`;
+  }
+
+  /**
+   * Call the OpenAI API to distill content from linked notes
+   * @param content The aggregated content from linked notes
+   * @returns Distilled content data
+   */
+  async distillContent(content: string): Promise<DistillResponse> {
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key not set');
+    }
+
+    const prompt = this.getDistillPrompt(content);
+    
+    try {
+      const response = await fetch('https://api.openai.com/v1/chat/completions', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({
+          model: 'gpt-4.1-2025-04-14',
+          messages: [{ role: 'user', content: prompt }],
+          temperature: 0.2,
+          max_tokens: 32768,
+          response_format: {
+            type: "json_schema",
+            json_schema: {
+              name: "distill_content",
+              schema: {
+                type: "object",
+                properties: {
+                  summary: {
+                    type: "string",
+                    description: "Concise summary that synthesizes the key concepts from all notes, with backlinks to atomic notes."
+                  },
+                  notes: {
+                    type: "array",
+                    items: {
+                      type: "object",
+                      properties: {
+                        title: {
+                          type: "string",
+                          description: "Title of the atomic note"
+                        },
+                        content: {
+                          type: "string",
+                          description: "Markdown-formatted, self-contained idea with backlinks if relevant"
+                        }
+                      },
+                      required: ["title", "content"],
+                      additionalProperties: false
+                    }
+                  },
+                  tasks: {
+                    type: "array",
+                    items: {
+                      type: "string",
+                      description: "Markdown-formatted task with checkbox"
+                    }
+                  }
+                },
+                required: ["summary", "notes", "tasks"],
+                additionalProperties: false
+              },
+              strict: true
+            },
+          }
+        })
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json();
+        throw new Error(`OpenAI API error: ${response.status} ${errorData.error?.message || response.statusText}`);
+      }
+
+      const responseData = await response.json();
+      const structuredData = responseData.choices[0].message.content;
+      
+      // Check for API refusal
+      if (responseData.choices[0].message.refusal) {
+        throw new Error(`API refusal: ${responseData.choices[0].message.refusal}`);
+      }
+      
+      // Parse the JSON
+      const parsedData: DistillResponse = typeof structuredData === 'string' 
+        ? JSON.parse(structuredData) 
+        : structuredData;
+        
+      // Initialize sourceNotes as empty array (will be populated by DistillService)
+      parsedData.sourceNotes = [];
+        
+      return parsedData;
+    } catch (error) {
+      console.error('Error calling OpenAI API:', error);
+      throw error;
+    }
+  }
 } 

src/types/transcript.ts

@@ -3,8 +3,14 @@ export interface TranscriptNote {
   content: string;
 }
 
-export interface TranscriptResponse {
+export interface BaseResponse {
   summary: string;
   notes: TranscriptNote[];
   tasks: string[];
+}
+
+export interface TranscriptResponse extends BaseResponse {}
+
+export interface DistillResponse extends BaseResponse {
+  sourceNotes: string[]; // List of source note names that were distilled
 }