Implement log file (#1804)

AGENTS.md

@@ -0,0 +1,280 @@
+# AGENTS.md
+
+This file provides guidance to any coding agent when working with code in this repository.
+
+## Overview
+
+Copilot for Obsidian is an AI-powered assistant plugin that integrates various LLM providers (OpenAI, Anthropic, Google, etc.) with Obsidian. It provides chat interfaces, autocomplete, semantic search, and various AI-powered commands for note-taking and knowledge management.
+
+## Development Commands
+
+### Build & Development
+
+- **NEVER RUN `npm run dev`** - The user will handle all builds manually
+- `npm run build` - Production build (TypeScript check + minified output)
+
+### Code Quality
+
+- `npm run lint` - Run ESLint checks
+- `npm run lint:fix` - Auto-fix ESLint issues
+- `npm run format` - Format code with Prettier
+- `npm run format:check` - Check formatting without changing files
+- **Before PR:** Always run `npm run format && npm run lint`
+
+### Testing
+
+- `npm run test` - Run unit tests (excludes integration tests)
+- `npm run test:integration` - Run integration tests (requires API keys)
+- Run single test: `npm test -- -t "test name"`
+
+## High-Level Architecture
+
+### Core Systems
+
+1. **LLM Provider System** (`src/LLMProviders/`)
+
+   - Provider implementations for OpenAI, Anthropic, Google, Azure, local models
+   - `LLMProviderManager` handles provider lifecycle and switching
+   - Stream-based responses with error handling and rate limiting
+   - Custom model configuration support
+
+2. **Chain Factory Pattern** (`src/chainFactory.ts`)
+
+   - Different chain types for various AI operations (chat, copilot, adhoc prompts)
+   - LangChain integration for complex workflows
+   - Memory management for conversation context
+   - Tool integration (search, file operations, time queries)
+
+3. **Vector Store & Search** (`src/search/`)
+
+   - `VectorStoreManager` manages embeddings and semantic search
+   - `ChunkedStorage` for efficient large document handling
+   - Event-driven index updates via `IndexManager`
+   - Multiple embedding providers support
+
+4. **UI Component System** (`src/components/`)
+
+   - React functional components with Radix UI primitives
+   - Tailwind CSS with class variance authority (CVA)
+   - Modal system for user interactions
+   - Chat interface with streaming support
+   - Settings UI with versioned components
+
+5. **Message Management Architecture** (`src/core/`, `src/state/`)
+
+   - **MessageRepository** (`src/core/MessageRepository.ts`): Single source of truth for all messages
+     - Stores each message once with both `displayText` and `processedText`
+     - Provides computed views for UI display and LLM processing
+     - No complex dual-array synchronization
+   - **ChatManager** (`src/core/ChatManager.ts`): Central business logic coordinator
+     - Orchestrates MessageRepository, ContextManager, and LLM operations
+     - Handles message sending, editing, regeneration, and deletion
+     - Manages context processing and chain memory synchronization
+     - **Project Chat Isolation**: Maintains separate MessageRepository per project
+       - Automatically detects project switches via `getCurrentMessageRepo()`
+       - Each project has its own isolated message history
+       - Non-project chats use `defaultProjectKey` repository
+   - **ChatUIState** (`src/state/ChatUIState.ts`): Clean UI-only state manager
+     - Delegates all business logic to ChatManager
+     - Provides React integration with subscription mechanism
+     - Replaces legacy SharedState with minimal, focused approach
+   - **ContextManager** (`src/core/ContextManager.ts`): Handles context processing
+     - Processes message context (notes, URLs, selected text)
+     - Reprocesses context when messages are edited
+
+6. **Settings Management**
+
+   - Jotai for atomic settings state management
+   - React contexts for feature-specific state
+
+7. **Plugin Integration**
+   - Main entry: `src/main.ts` extends Obsidian Plugin
+   - Command registration system
+   - Event handling for Obsidian lifecycle
+   - Settings persistence and migration
+   - Chat history loading via pending message mechanism
+
+### Key Patterns
+
+- **Single Source of Truth**: MessageRepository stores each message once with computed views
+- **Clean Architecture**: Repository → Manager → UIState → React Components
+- **Context Reprocessing**: Automatic context updates when messages are edited
+- **Computed Views**: Display messages for UI, LLM messages for AI processing
+- **Project Isolation**: Each project maintains its own MessageRepository instance
+- **Error Handling**: Custom error types with detailed interfaces
+- **Async Operations**: Consistent async/await pattern with proper error boundaries
+- **Caching**: Multi-layer caching for files, PDFs, and API responses
+- **Streaming**: Real-time streaming for LLM responses
+- **Testing**: Unit tests adjacent to implementation, integration tests for API calls
+
+## Message Management Architecture
+
+For detailed architecture diagrams and documentation, see [`MESSAGE_ARCHITECTURE.md`](./MESSAGE_ARCHITECTURE.md).
+
+### Core Classes and Flow
+
+1. **MessageRepository** (`src/core/MessageRepository.ts`)
+
+   - Single source of truth for all messages
+   - Stores `StoredMessage` objects with both `displayText` and `processedText`
+   - Provides computed views via `getDisplayMessages()` and `getLLMMessages()`
+   - No complex dual-array synchronization or ID matching
+
+2. **ChatManager** (`src/core/ChatManager.ts`)
+
+   - Central business logic coordinator
+   - Orchestrates MessageRepository, ContextManager, and LLM operations
+   - Handles all message CRUD operations with proper error handling
+   - Synchronizes with chain memory for conversation history
+   - **Project Chat Isolation Implementation**:
+     - Maintains `projectMessageRepos: Map<string, MessageRepository>` for project-specific storage
+     - `getCurrentMessageRepo()` automatically detects current project and returns correct repository
+     - Seamlessly switches between project repositories when project changes
+     - Creates new empty repository for each project (no message caching)
+
+3. **ChatUIState** (`src/state/ChatUIState.ts`)
+
+   - Clean UI-only state manager
+   - Delegates all business logic to ChatManager
+   - Provides React integration with subscription mechanism
+   - Replaces legacy SharedState with minimal, focused approach
+
+4. **ContextManager** (`src/core/ContextManager.ts`)
+
+   - Handles context processing (notes, URLs, selected text)
+   - Reprocesses context when messages are edited
+   - Ensures fresh context for LLM processing
+
+5. **ChatPersistenceManager** (`src/core/ChatPersistenceManager.ts`)
+   - Handles saving and loading chat history to/from markdown files
+   - Project-aware file naming (prefixes with project ID)
+   - Parses and formats chat content for storage
+   - Integrated with ChatManager for seamless persistence
+
+## Code Style Guidelines
+
+### MAJOR PRINCIPLES
+
+- **ALWAYS WRITE GENERALIZABLE SOLUTIONS**: Never add edge-case handling or hardcoded logic for specific scenarios (like "piano notes" or "daily notes"). Solutions must work for all cases.
+- **NEVER MODIFY AI PROMPT CONTENT**: Do not update, edit, or change any AI prompts, system prompts, or model adapter prompts unless explicitly asked to do so by the user
+- **Avoid hardcoding**: No hardcoded folder names, file patterns, or special-case logic
+- **Configuration over convention**: If behavior needs to vary, make it configurable, not hardcoded
+- **Universal patterns**: Solutions should work equally well for any folder structure, naming convention, or content type
+
+### TypeScript
+
+- Strict mode enabled (no implicit any, strict null checks)
+- Use absolute imports with `@/` prefix: `import { ChainType } from "@/chainFactory"`
+- Prefer const assertions and type inference where appropriate
+- Use interface for object shapes, type for unions/aliases
+
+### React
+
+- Functional components only (no class components)
+- Custom hooks for reusable logic
+- Props interfaces defined above components
+- Avoid inline styles, use Tailwind classes
+
+### General
+
+- File naming: PascalCase for components, camelCase for utilities
+- Async/await over promises
+- Early returns for error conditions
+- **Always add JSDoc comments** for all functions and methods
+- Organize imports: React → external → internal
+- **Avoid language-specific lists** (like stopwords or action verbs) - use language-agnostic approaches instead
+
+### Logging
+
+- **NEVER use console.log** - Use the logging utilities instead:
+  - `logInfo()` for informational messages
+  - `logWarn()` for warnings
+  - `logError()` for errors
+- Import from logger: `import { logInfo, logWarn, logError } from "@/logger"`
+
+## Testing Guidelines
+
+- Unit tests use Jest with TypeScript support
+- Mock Obsidian API for plugin testing
+- Integration tests require API keys in `.env.test`
+- Test files adjacent to implementation (`.test.ts`)
+- Use `@testing-library/react` for component testing
+
+## Development Session Planning
+
+### Using TODO.md for Session Management
+
+**IMPORTANT**: When working on a development session, maintain a comprehensive `TODO.md` file that serves as the central plan and tracker:
+
+1. **Session Goal**: Define the high-level objective at the start
+2. **Task Tracking**:
+   - List all completed tasks with [x] checkboxes
+   - Track pending tasks with [ ] checkboxes
+   - Group related tasks into logical sections
+3. **Architecture Decisions**: Document key design choices and rationale
+4. **Progress Updates**: Keep the TODO.md updated as tasks complete
+5. **Testing Checklist**: Include verification steps for the session
+
+The TODO.md should be:
+
+- The single source of truth for session progress
+- Updated frequently as work progresses
+- Clear enough that another developer can understand what was done
+- Comprehensive enough to serve as a migration guide
+
+### Structure Example:
+
+```markdown
+# Development Session TODO
+
+## Session Goal
+
+[Clear statement of what this session aims to achieve]
+
+## Completed Tasks ✅
+
+- [x] Task description with key details
+- [x] Another completed task
+
+## Pending Tasks 📋
+
+- [ ] Next task to work on
+- [ ] Future enhancement
+
+## Architecture Summary
+
+[Key design decisions and rationale]
+
+## Testing Checklist
+
+- [ ] Functionality verification
+- [ ] Performance checks
+```
+
+## Important Notes
+
+- The plugin supports multiple LLM providers with custom endpoints
+- Vector store requires rebuilding when switching embedding providers
+- Settings are versioned - migrations may be needed
+- Local model support available via Ollama/LM Studio
+- Rate limiting is implemented for all API calls
+- For technical debt and known issues, see [`TECHDEBT.md`](./TECHDEBT.md)
+- For current development session planning, see [`TODO.md`](./TODO.md)
+
+### Obsidian Plugin Environment
+
+- **Global `app` variable**: In Obsidian plugins, `app` is a globally available variable that provides access to the Obsidian API. It's automatically available in all files without needing to import or declare it.
+
+### Architecture Migration Notes
+
+- **SharedState Removed**: The legacy `src/sharedState.ts` has been completely removed
+- **Clean Architecture**: New architecture follows Repository → Manager → UIState → UI pattern
+- **Single Source of Truth**: All messages stored once in MessageRepository with computed views
+- **Context Always Fresh**: Context is reprocessed when messages are edited to ensure accuracy
+- **Chat History Loading**: Uses pending message mechanism through CopilotView → Chat component props
+- **Project Chat Isolation**: Each project now has completely isolated chat history
+  - Automatic detection of project switches via `ProjectManager.getCurrentProjectId()`
+  - Separate MessageRepository instances per project ID
+  - Non-project chats stored in default repository
+  - Backwards compatible - loads existing messages from ProjectManager cache
+  - Zero configuration required - works automatically

src/errorFormat.ts

@@ -0,0 +1,21 @@
+export function err2String(err: unknown, stack = false): string {
+  try {
+    if (err instanceof Error) {
+      const causeMsg =
+        (err as any)?.cause instanceof Error
+          ? (err as any).cause.message
+          : (err as any)?.cause
+            ? String((err as any).cause)
+            : "";
+      const stackStr = stack && err.stack ? err.stack : "";
+      const parts = [err.message];
+      if (causeMsg) parts.push(`more message: ${causeMsg}`);
+      if (stackStr) parts.push(stackStr);
+      return parts.join("\n");
+    }
+    const json = JSON.stringify(err);
+    return json ?? String(err);
+  } catch {
+    return String(err);
+  }
+}

src/logFileManager.ts

@@ -0,0 +1,176 @@
+import { err2String } from "@/errorFormat";
+import { TFile } from "obsidian";
+
+type LogLevel = "INFO" | "WARN" | "ERROR";
+
+/**
+ * Manages a rolling log file that keeps the last N entries and works on desktop and mobile.
+ * - Writes to <vault>/copilot-log.md
+ * - Maintains an in-memory ring buffer of the last 1000 entries
+ * - Debounced flush to reduce I/O; single-line entries to preserve accurate line limits
+ */
+class LogFileManager {
+  private static instance: LogFileManager;
+
+  private readonly maxLines = 1000;
+  private readonly debounceMs = 1500; // per user preference
+  private readonly maxLineChars = 8000; // guard against extremely large entries
+  private buffer: string[] = [];
+  private initialized = false;
+  private flushing = false;
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+
+  static getInstance(): LogFileManager {
+    if (!LogFileManager.instance) {
+      LogFileManager.instance = new LogFileManager();
+    }
+    return LogFileManager.instance;
+  }
+
+  getLogPath(): string {
+    return "copilot-log.md"; // vault root
+  }
+
+  /** Ensure buffer is loaded with up to last 1000 lines from existing file. */
+  private async ensureInitialized() {
+    if (this.initialized) return;
+    try {
+      if (!this.hasVault()) {
+        this.initialized = true;
+        return;
+      }
+      const path = this.getLogPath();
+      const exists = await app.vault.adapter.exists(path);
+      if (exists) {
+        const content = await app.vault.adapter.read(path);
+        // Normalize line endings and split
+        const lines = content.split(/\r?\n/).filter((l) => l.length > 0);
+        if (lines.length > this.maxLines) {
+          this.buffer = lines.slice(lines.length - this.maxLines);
+        } else {
+          this.buffer = lines;
+        }
+      }
+    } catch {
+      // Ignore read errors; start fresh
+      this.buffer = [];
+    } finally {
+      this.initialized = true;
+    }
+  }
+
+  private hasVault(): boolean {
+    // global `app` is available in Obsidian environment
+    try {
+      return typeof app !== "undefined" && !!app.vault?.adapter;
+    } catch {
+      return false;
+    }
+  }
+
+  private sanitizeForSingleLine(value: unknown): string {
+    // Error handling: include stack traces by default as requested, collapsed to one line
+    if (value instanceof Error) {
+      const withStack = err2String(value, true);
+      return this.collapseToSingleLine(withStack);
+    }
+
+    if (typeof value === "string") {
+      return this.collapseToSingleLine(value);
+    }
+
+    // JSON stringify without spacing; fall back to String()
+    try {
+      const json = JSON.stringify(value);
+      return this.collapseToSingleLine(json ?? String(value));
+    } catch {
+      return this.collapseToSingleLine(String(value));
+    }
+  }
+
+  private collapseToSingleLine(s: string): string {
+    // Replace CR/LF and tabs to keep a single physical line in the log file
+    const oneLine = s.replace(/[\r\n]+/g, "\\n").replace(/\t/g, " ");
+    if (oneLine.length <= this.maxLineChars) return oneLine;
+    return (
+      oneLine.slice(0, this.maxLineChars) +
+      ` … [truncated ${oneLine.length - this.maxLineChars} chars]`
+    );
+  }
+
+  async append(level: LogLevel, ...args: unknown[]) {
+    await this.ensureInitialized();
+
+    const ts = new Date().toISOString();
+    const parts = args.map((a) => this.sanitizeForSingleLine(a));
+    const line = `${ts} ${level} ${parts.join(" ")}`.trim();
+
+    this.buffer.push(line);
+    if (this.buffer.length > this.maxLines) {
+      this.buffer.splice(0, this.buffer.length - this.maxLines);
+    }
+
+    this.scheduleFlush();
+  }
+
+  private scheduleFlush() {
+    if (!this.hasVault()) return; // no-op in tests or non-Obsidian env
+    if (this.flushTimer !== null) {
+      clearTimeout(this.flushTimer);
+    }
+    this.flushTimer = setTimeout(() => {
+      this.flushTimer = null;
+      void this.flush();
+    }, this.debounceMs);
+  }
+
+  async flush(): Promise<void> {
+    if (!this.hasVault()) return;
+    if (this.flushing) return;
+    this.flushing = true;
+    try {
+      const path = this.getLogPath();
+      const content = this.buffer.join("\n") + (this.buffer.length ? "\n" : "");
+      await app.vault.adapter.write(path, content);
+    } catch {
+      // swallow write errors; logging should never crash the app
+    } finally {
+      this.flushing = false;
+    }
+  }
+
+  async clear(): Promise<void> {
+    this.buffer = [];
+    if (!this.hasVault()) return;
+    try {
+      const path = this.getLogPath();
+      if (await app.vault.adapter.exists(path)) {
+        await app.vault.adapter.write(path, "");
+      }
+    } catch {
+      // ignore
+    }
+  }
+
+  async openLogFile(): Promise<void> {
+    await this.flush();
+    if (!this.hasVault()) return;
+    const path = this.getLogPath();
+    let file = app.vault.getAbstractFileByPath(path) as TFile | null;
+    try {
+      if (!file) {
+        // Create file if missing so it can be opened
+        file = await app.vault.create(
+          path,
+          this.buffer.join("\n") + (this.buffer.length ? "\n" : "")
+        );
+      }
+      const leaf = app.workspace.getLeaf(true);
+      await leaf.openFile(file);
+    } catch {
+      // ignore
+    }
+  }
+}
+
+export const logFileManager = LogFileManager.getInstance();

src/logger.ts

@@ -1,19 +1,25 @@
 import { getSettings } from "@/settings/model";
+import { logFileManager } from "@/logFileManager";
 
 export function logInfo(...args: any[]) {
   if (getSettings().debug) {
     console.log(...args);
   }
+  // Always append to rolling file log
+  void logFileManager.append("INFO", ...args);
 }
 
 export function logError(...args: any[]) {
+  // Always include stack traces by default; console logs still respect debug
   if (getSettings().debug) {
     console.error(...args);
   }
+  void logFileManager.append("ERROR", ...args);
 }
 
 export function logWarn(...args: any[]) {
   if (getSettings().debug) {
     console.warn(...args);
   }
+  void logFileManager.append("WARN", ...args);
 }

src/main.ts

@@ -18,6 +18,7 @@ import { ChatManager } from "@/core/ChatManager";
 import { MessageRepository } from "@/core/MessageRepository";
 import { encryptAllKeys } from "@/encryptionService";
 import { logInfo } from "@/logger";
+import { logFileManager } from "@/logFileManager";
 import { checkIsPlusUser } from "@/plusUtils";
 import VectorStoreManager from "@/search/vectorStoreManager";
 import { CopilotSettingTab } from "@/settings/SettingsPage";
@@ -162,6 +163,8 @@ export default class CopilotPlugin extends Plugin {
     this.settingsUnsubscriber?.();
     this.autocompleteService?.destroy();
 
+    // Best-effort flush of log file
+    await logFileManager.flush();
     logInfo("Copilot plugin unloaded");
   }
 

src/settings/v2/components/AdvancedSettings.tsx

@@ -1,4 +1,6 @@
 import { SettingItem } from "@/components/ui/setting-item";
+import { Button } from "@/components/ui/button";
+import { logFileManager } from "@/logFileManager";
 import { updateSetting, useSettingsValue } from "@/settings/model";
 import React from "react";
 
@@ -38,6 +40,23 @@ export const AdvancedSettings: React.FC = () => {
               updateSetting("debug", checked);
             }}
           />
+
+          <SettingItem
+            type="custom"
+            title="Share Log"
+            description="Open the Copilot log file (copilot-log.md) for easy sharing."
+          >
+            <Button
+              variant="secondary"
+              size="sm"
+              onClick={async () => {
+                await logFileManager.flush();
+                await logFileManager.openLogFile();
+              }}
+            >
+              Open Log File
+            </Button>
+          </SettingItem>
         </div>
       </section>
     </div>

src/utils.ts

@@ -20,6 +20,7 @@ import { BaseChain, RetrievalQAChain } from "langchain/chains";
 import moment from "moment";
 import { MarkdownView, Notice, TFile, Vault, requestUrl } from "obsidian";
 import { CustomModel } from "./aiParams";
+export { err2String } from "@/errorFormat";
 
 // Add custom error type at the top of the file
 interface APIError extends Error {
@@ -654,16 +655,7 @@ function createReadableStreamFromString(input: string) {
   });
 }
 
-export function err2String(err: any, stack = false) {
-  // maybe to be improved
-  return err instanceof Error
-    ? err.message +
-        "\n" +
-        `${err?.cause ? "more message: " + (err.cause as Error).message : ""}` +
-        "\n" +
-        `${stack ? err.stack : ""}`
-    : JSON.stringify(err);
-}
+// err2String is now exported from '@/errorFormat' to avoid circular dependencies and duplication.
 
 export function omit<T extends Record<string, any>, K extends keyof T>(
   obj: T,