From c033655b96dffaed6860195a442ae7e64e333d6f Mon Sep 17 00:00:00 2001 From: Logan Yang Date: Wed, 10 Sep 2025 22:34:57 -0700 Subject: [PATCH] Implement log file (#1804) --- AGENTS.md | 280 ++++++++++++++++++ src/errorFormat.ts | 21 ++ src/logFileManager.ts | 176 +++++++++++ src/logger.ts | 6 + src/main.ts | 3 + .../v2/components/AdvancedSettings.tsx | 19 ++ src/utils.ts | 12 +- 7 files changed, 507 insertions(+), 10 deletions(-) create mode 100644 AGENTS.md create mode 100644 src/errorFormat.ts create mode 100644 src/logFileManager.ts diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..2783994d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,280 @@ +# AGENTS.md + +This file provides guidance to any coding agent when working with code in this repository. + +## Overview + +Copilot for Obsidian is an AI-powered assistant plugin that integrates various LLM providers (OpenAI, Anthropic, Google, etc.) with Obsidian. It provides chat interfaces, autocomplete, semantic search, and various AI-powered commands for note-taking and knowledge management. + +## Development Commands + +### Build & Development + +- **NEVER RUN `npm run dev`** - The user will handle all builds manually +- `npm run build` - Production build (TypeScript check + minified output) + +### Code Quality + +- `npm run lint` - Run ESLint checks +- `npm run lint:fix` - Auto-fix ESLint issues +- `npm run format` - Format code with Prettier +- `npm run format:check` - Check formatting without changing files +- **Before PR:** Always run `npm run format && npm run lint` + +### Testing + +- `npm run test` - Run unit tests (excludes integration tests) +- `npm run test:integration` - Run integration tests (requires API keys) +- Run single test: `npm test -- -t "test name"` + +## High-Level Architecture + +### Core Systems + +1. **LLM Provider System** (`src/LLMProviders/`) + + - Provider implementations for OpenAI, Anthropic, Google, Azure, local models + - `LLMProviderManager` handles provider lifecycle and switching + - Stream-based responses with error handling and rate limiting + - Custom model configuration support + +2. **Chain Factory Pattern** (`src/chainFactory.ts`) + + - Different chain types for various AI operations (chat, copilot, adhoc prompts) + - LangChain integration for complex workflows + - Memory management for conversation context + - Tool integration (search, file operations, time queries) + +3. **Vector Store & Search** (`src/search/`) + + - `VectorStoreManager` manages embeddings and semantic search + - `ChunkedStorage` for efficient large document handling + - Event-driven index updates via `IndexManager` + - Multiple embedding providers support + +4. **UI Component System** (`src/components/`) + + - React functional components with Radix UI primitives + - Tailwind CSS with class variance authority (CVA) + - Modal system for user interactions + - Chat interface with streaming support + - Settings UI with versioned components + +5. **Message Management Architecture** (`src/core/`, `src/state/`) + + - **MessageRepository** (`src/core/MessageRepository.ts`): Single source of truth for all messages + - Stores each message once with both `displayText` and `processedText` + - Provides computed views for UI display and LLM processing + - No complex dual-array synchronization + - **ChatManager** (`src/core/ChatManager.ts`): Central business logic coordinator + - Orchestrates MessageRepository, ContextManager, and LLM operations + - Handles message sending, editing, regeneration, and deletion + - Manages context processing and chain memory synchronization + - **Project Chat Isolation**: Maintains separate MessageRepository per project + - Automatically detects project switches via `getCurrentMessageRepo()` + - Each project has its own isolated message history + - Non-project chats use `defaultProjectKey` repository + - **ChatUIState** (`src/state/ChatUIState.ts`): Clean UI-only state manager + - Delegates all business logic to ChatManager + - Provides React integration with subscription mechanism + - Replaces legacy SharedState with minimal, focused approach + - **ContextManager** (`src/core/ContextManager.ts`): Handles context processing + - Processes message context (notes, URLs, selected text) + - Reprocesses context when messages are edited + +6. **Settings Management** + + - Jotai for atomic settings state management + - React contexts for feature-specific state + +7. **Plugin Integration** + - Main entry: `src/main.ts` extends Obsidian Plugin + - Command registration system + - Event handling for Obsidian lifecycle + - Settings persistence and migration + - Chat history loading via pending message mechanism + +### Key Patterns + +- **Single Source of Truth**: MessageRepository stores each message once with computed views +- **Clean Architecture**: Repository → Manager → UIState → React Components +- **Context Reprocessing**: Automatic context updates when messages are edited +- **Computed Views**: Display messages for UI, LLM messages for AI processing +- **Project Isolation**: Each project maintains its own MessageRepository instance +- **Error Handling**: Custom error types with detailed interfaces +- **Async Operations**: Consistent async/await pattern with proper error boundaries +- **Caching**: Multi-layer caching for files, PDFs, and API responses +- **Streaming**: Real-time streaming for LLM responses +- **Testing**: Unit tests adjacent to implementation, integration tests for API calls + +## Message Management Architecture + +For detailed architecture diagrams and documentation, see [`MESSAGE_ARCHITECTURE.md`](./MESSAGE_ARCHITECTURE.md). + +### Core Classes and Flow + +1. **MessageRepository** (`src/core/MessageRepository.ts`) + + - Single source of truth for all messages + - Stores `StoredMessage` objects with both `displayText` and `processedText` + - Provides computed views via `getDisplayMessages()` and `getLLMMessages()` + - No complex dual-array synchronization or ID matching + +2. **ChatManager** (`src/core/ChatManager.ts`) + + - Central business logic coordinator + - Orchestrates MessageRepository, ContextManager, and LLM operations + - Handles all message CRUD operations with proper error handling + - Synchronizes with chain memory for conversation history + - **Project Chat Isolation Implementation**: + - Maintains `projectMessageRepos: Map` for project-specific storage + - `getCurrentMessageRepo()` automatically detects current project and returns correct repository + - Seamlessly switches between project repositories when project changes + - Creates new empty repository for each project (no message caching) + +3. **ChatUIState** (`src/state/ChatUIState.ts`) + + - Clean UI-only state manager + - Delegates all business logic to ChatManager + - Provides React integration with subscription mechanism + - Replaces legacy SharedState with minimal, focused approach + +4. **ContextManager** (`src/core/ContextManager.ts`) + + - Handles context processing (notes, URLs, selected text) + - Reprocesses context when messages are edited + - Ensures fresh context for LLM processing + +5. **ChatPersistenceManager** (`src/core/ChatPersistenceManager.ts`) + - Handles saving and loading chat history to/from markdown files + - Project-aware file naming (prefixes with project ID) + - Parses and formats chat content for storage + - Integrated with ChatManager for seamless persistence + +## Code Style Guidelines + +### MAJOR PRINCIPLES + +- **ALWAYS WRITE GENERALIZABLE SOLUTIONS**: Never add edge-case handling or hardcoded logic for specific scenarios (like "piano notes" or "daily notes"). Solutions must work for all cases. +- **NEVER MODIFY AI PROMPT CONTENT**: Do not update, edit, or change any AI prompts, system prompts, or model adapter prompts unless explicitly asked to do so by the user +- **Avoid hardcoding**: No hardcoded folder names, file patterns, or special-case logic +- **Configuration over convention**: If behavior needs to vary, make it configurable, not hardcoded +- **Universal patterns**: Solutions should work equally well for any folder structure, naming convention, or content type + +### TypeScript + +- Strict mode enabled (no implicit any, strict null checks) +- Use absolute imports with `@/` prefix: `import { ChainType } from "@/chainFactory"` +- Prefer const assertions and type inference where appropriate +- Use interface for object shapes, type for unions/aliases + +### React + +- Functional components only (no class components) +- Custom hooks for reusable logic +- Props interfaces defined above components +- Avoid inline styles, use Tailwind classes + +### General + +- File naming: PascalCase for components, camelCase for utilities +- Async/await over promises +- Early returns for error conditions +- **Always add JSDoc comments** for all functions and methods +- Organize imports: React → external → internal +- **Avoid language-specific lists** (like stopwords or action verbs) - use language-agnostic approaches instead + +### Logging + +- **NEVER use console.log** - Use the logging utilities instead: + - `logInfo()` for informational messages + - `logWarn()` for warnings + - `logError()` for errors +- Import from logger: `import { logInfo, logWarn, logError } from "@/logger"` + +## Testing Guidelines + +- Unit tests use Jest with TypeScript support +- Mock Obsidian API for plugin testing +- Integration tests require API keys in `.env.test` +- Test files adjacent to implementation (`.test.ts`) +- Use `@testing-library/react` for component testing + +## Development Session Planning + +### Using TODO.md for Session Management + +**IMPORTANT**: When working on a development session, maintain a comprehensive `TODO.md` file that serves as the central plan and tracker: + +1. **Session Goal**: Define the high-level objective at the start +2. **Task Tracking**: + - List all completed tasks with [x] checkboxes + - Track pending tasks with [ ] checkboxes + - Group related tasks into logical sections +3. **Architecture Decisions**: Document key design choices and rationale +4. **Progress Updates**: Keep the TODO.md updated as tasks complete +5. **Testing Checklist**: Include verification steps for the session + +The TODO.md should be: + +- The single source of truth for session progress +- Updated frequently as work progresses +- Clear enough that another developer can understand what was done +- Comprehensive enough to serve as a migration guide + +### Structure Example: + +```markdown +# Development Session TODO + +## Session Goal + +[Clear statement of what this session aims to achieve] + +## Completed Tasks ✅ + +- [x] Task description with key details +- [x] Another completed task + +## Pending Tasks 📋 + +- [ ] Next task to work on +- [ ] Future enhancement + +## Architecture Summary + +[Key design decisions and rationale] + +## Testing Checklist + +- [ ] Functionality verification +- [ ] Performance checks +``` + +## Important Notes + +- The plugin supports multiple LLM providers with custom endpoints +- Vector store requires rebuilding when switching embedding providers +- Settings are versioned - migrations may be needed +- Local model support available via Ollama/LM Studio +- Rate limiting is implemented for all API calls +- For technical debt and known issues, see [`TECHDEBT.md`](./TECHDEBT.md) +- For current development session planning, see [`TODO.md`](./TODO.md) + +### Obsidian Plugin Environment + +- **Global `app` variable**: In Obsidian plugins, `app` is a globally available variable that provides access to the Obsidian API. It's automatically available in all files without needing to import or declare it. + +### Architecture Migration Notes + +- **SharedState Removed**: The legacy `src/sharedState.ts` has been completely removed +- **Clean Architecture**: New architecture follows Repository → Manager → UIState → UI pattern +- **Single Source of Truth**: All messages stored once in MessageRepository with computed views +- **Context Always Fresh**: Context is reprocessed when messages are edited to ensure accuracy +- **Chat History Loading**: Uses pending message mechanism through CopilotView → Chat component props +- **Project Chat Isolation**: Each project now has completely isolated chat history + - Automatic detection of project switches via `ProjectManager.getCurrentProjectId()` + - Separate MessageRepository instances per project ID + - Non-project chats stored in default repository + - Backwards compatible - loads existing messages from ProjectManager cache + - Zero configuration required - works automatically diff --git a/src/errorFormat.ts b/src/errorFormat.ts new file mode 100644 index 00000000..e606ca7e --- /dev/null +++ b/src/errorFormat.ts @@ -0,0 +1,21 @@ +export function err2String(err: unknown, stack = false): string { + try { + if (err instanceof Error) { + const causeMsg = + (err as any)?.cause instanceof Error + ? (err as any).cause.message + : (err as any)?.cause + ? String((err as any).cause) + : ""; + const stackStr = stack && err.stack ? err.stack : ""; + const parts = [err.message]; + if (causeMsg) parts.push(`more message: ${causeMsg}`); + if (stackStr) parts.push(stackStr); + return parts.join("\n"); + } + const json = JSON.stringify(err); + return json ?? String(err); + } catch { + return String(err); + } +} diff --git a/src/logFileManager.ts b/src/logFileManager.ts new file mode 100644 index 00000000..89e7fa6b --- /dev/null +++ b/src/logFileManager.ts @@ -0,0 +1,176 @@ +import { err2String } from "@/errorFormat"; +import { TFile } from "obsidian"; + +type LogLevel = "INFO" | "WARN" | "ERROR"; + +/** + * Manages a rolling log file that keeps the last N entries and works on desktop and mobile. + * - Writes to /copilot-log.md + * - Maintains an in-memory ring buffer of the last 1000 entries + * - Debounced flush to reduce I/O; single-line entries to preserve accurate line limits + */ +class LogFileManager { + private static instance: LogFileManager; + + private readonly maxLines = 1000; + private readonly debounceMs = 1500; // per user preference + private readonly maxLineChars = 8000; // guard against extremely large entries + private buffer: string[] = []; + private initialized = false; + private flushing = false; + private flushTimer: ReturnType | null = null; + + static getInstance(): LogFileManager { + if (!LogFileManager.instance) { + LogFileManager.instance = new LogFileManager(); + } + return LogFileManager.instance; + } + + getLogPath(): string { + return "copilot-log.md"; // vault root + } + + /** Ensure buffer is loaded with up to last 1000 lines from existing file. */ + private async ensureInitialized() { + if (this.initialized) return; + try { + if (!this.hasVault()) { + this.initialized = true; + return; + } + const path = this.getLogPath(); + const exists = await app.vault.adapter.exists(path); + if (exists) { + const content = await app.vault.adapter.read(path); + // Normalize line endings and split + const lines = content.split(/\r?\n/).filter((l) => l.length > 0); + if (lines.length > this.maxLines) { + this.buffer = lines.slice(lines.length - this.maxLines); + } else { + this.buffer = lines; + } + } + } catch { + // Ignore read errors; start fresh + this.buffer = []; + } finally { + this.initialized = true; + } + } + + private hasVault(): boolean { + // global `app` is available in Obsidian environment + try { + return typeof app !== "undefined" && !!app.vault?.adapter; + } catch { + return false; + } + } + + private sanitizeForSingleLine(value: unknown): string { + // Error handling: include stack traces by default as requested, collapsed to one line + if (value instanceof Error) { + const withStack = err2String(value, true); + return this.collapseToSingleLine(withStack); + } + + if (typeof value === "string") { + return this.collapseToSingleLine(value); + } + + // JSON stringify without spacing; fall back to String() + try { + const json = JSON.stringify(value); + return this.collapseToSingleLine(json ?? String(value)); + } catch { + return this.collapseToSingleLine(String(value)); + } + } + + private collapseToSingleLine(s: string): string { + // Replace CR/LF and tabs to keep a single physical line in the log file + const oneLine = s.replace(/[\r\n]+/g, "\\n").replace(/\t/g, " "); + if (oneLine.length <= this.maxLineChars) return oneLine; + return ( + oneLine.slice(0, this.maxLineChars) + + ` … [truncated ${oneLine.length - this.maxLineChars} chars]` + ); + } + + async append(level: LogLevel, ...args: unknown[]) { + await this.ensureInitialized(); + + const ts = new Date().toISOString(); + const parts = args.map((a) => this.sanitizeForSingleLine(a)); + const line = `${ts} ${level} ${parts.join(" ")}`.trim(); + + this.buffer.push(line); + if (this.buffer.length > this.maxLines) { + this.buffer.splice(0, this.buffer.length - this.maxLines); + } + + this.scheduleFlush(); + } + + private scheduleFlush() { + if (!this.hasVault()) return; // no-op in tests or non-Obsidian env + if (this.flushTimer !== null) { + clearTimeout(this.flushTimer); + } + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + void this.flush(); + }, this.debounceMs); + } + + async flush(): Promise { + if (!this.hasVault()) return; + if (this.flushing) return; + this.flushing = true; + try { + const path = this.getLogPath(); + const content = this.buffer.join("\n") + (this.buffer.length ? "\n" : ""); + await app.vault.adapter.write(path, content); + } catch { + // swallow write errors; logging should never crash the app + } finally { + this.flushing = false; + } + } + + async clear(): Promise { + this.buffer = []; + if (!this.hasVault()) return; + try { + const path = this.getLogPath(); + if (await app.vault.adapter.exists(path)) { + await app.vault.adapter.write(path, ""); + } + } catch { + // ignore + } + } + + async openLogFile(): Promise { + await this.flush(); + if (!this.hasVault()) return; + const path = this.getLogPath(); + let file = app.vault.getAbstractFileByPath(path) as TFile | null; + try { + if (!file) { + // Create file if missing so it can be opened + file = await app.vault.create( + path, + this.buffer.join("\n") + (this.buffer.length ? "\n" : "") + ); + } + const leaf = app.workspace.getLeaf(true); + await leaf.openFile(file); + } catch { + // ignore + } + } +} + +export const logFileManager = LogFileManager.getInstance(); diff --git a/src/logger.ts b/src/logger.ts index 2566516a..53f5320f 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,19 +1,25 @@ import { getSettings } from "@/settings/model"; +import { logFileManager } from "@/logFileManager"; export function logInfo(...args: any[]) { if (getSettings().debug) { console.log(...args); } + // Always append to rolling file log + void logFileManager.append("INFO", ...args); } export function logError(...args: any[]) { + // Always include stack traces by default; console logs still respect debug if (getSettings().debug) { console.error(...args); } + void logFileManager.append("ERROR", ...args); } export function logWarn(...args: any[]) { if (getSettings().debug) { console.warn(...args); } + void logFileManager.append("WARN", ...args); } diff --git a/src/main.ts b/src/main.ts index 90da3b82..2af90aad 100644 --- a/src/main.ts +++ b/src/main.ts @@ -18,6 +18,7 @@ import { ChatManager } from "@/core/ChatManager"; import { MessageRepository } from "@/core/MessageRepository"; import { encryptAllKeys } from "@/encryptionService"; import { logInfo } from "@/logger"; +import { logFileManager } from "@/logFileManager"; import { checkIsPlusUser } from "@/plusUtils"; import VectorStoreManager from "@/search/vectorStoreManager"; import { CopilotSettingTab } from "@/settings/SettingsPage"; @@ -162,6 +163,8 @@ export default class CopilotPlugin extends Plugin { this.settingsUnsubscriber?.(); this.autocompleteService?.destroy(); + // Best-effort flush of log file + await logFileManager.flush(); logInfo("Copilot plugin unloaded"); } diff --git a/src/settings/v2/components/AdvancedSettings.tsx b/src/settings/v2/components/AdvancedSettings.tsx index 3fe46adc..cf49c4b2 100644 --- a/src/settings/v2/components/AdvancedSettings.tsx +++ b/src/settings/v2/components/AdvancedSettings.tsx @@ -1,4 +1,6 @@ import { SettingItem } from "@/components/ui/setting-item"; +import { Button } from "@/components/ui/button"; +import { logFileManager } from "@/logFileManager"; import { updateSetting, useSettingsValue } from "@/settings/model"; import React from "react"; @@ -38,6 +40,23 @@ export const AdvancedSettings: React.FC = () => { updateSetting("debug", checked); }} /> + + + + diff --git a/src/utils.ts b/src/utils.ts index a5e7672d..56052cf9 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -20,6 +20,7 @@ import { BaseChain, RetrievalQAChain } from "langchain/chains"; import moment from "moment"; import { MarkdownView, Notice, TFile, Vault, requestUrl } from "obsidian"; import { CustomModel } from "./aiParams"; +export { err2String } from "@/errorFormat"; // Add custom error type at the top of the file interface APIError extends Error { @@ -654,16 +655,7 @@ function createReadableStreamFromString(input: string) { }); } -export function err2String(err: any, stack = false) { - // maybe to be improved - return err instanceof Error - ? err.message + - "\n" + - `${err?.cause ? "more message: " + (err.cause as Error).message : ""}` + - "\n" + - `${stack ? err.stack : ""}` - : JSON.stringify(err); -} +// err2String is now exported from '@/errorFormat' to avoid circular dependencies and duplication. export function omit, K extends keyof T>( obj: T,