diff --git a/docs/miyo-api.md b/docs/miyo-api.md new file mode 100644 index 00000000..046b25d2 --- /dev/null +++ b/docs/miyo-api.md @@ -0,0 +1,471 @@ +# Miyo Node Service API + +Base URL: `http://127.0.0.1:8742` + +All request and response bodies are JSON. Errors always return `{ "detail": "" }`. + +--- + +## Health + +### `GET /v0/health` + +Returns service and sidecar status. + +**Response 200** + +```json +{ + "status": "ok | degraded", + "service": "running", + "qdrant": "connected | ...", + "llama_server": "running | ...", + "model_download_progress": 0.75, + "embedding_model": "nomic-embed-text-v1.5", + "batch_size_preset": "default", + "gpu_variant": "metal | null", + "indexed_files": 1234 +} +``` + +--- + +## Search + +### `POST /v0/search` + +Hybrid semantic + keyword search (dense + BM25, fused via RRF). + +**Request body** + +```json +{ + "query": "string (required)", + "folder_path": "string | null — restrict to this folder", + "path": "string | null — substring filter on file path (case-insensitive)", + "limit": 10, + "filters": [ + /* MetadataFilter[], see below */ + ] +} +``` + +**Response 200** + +```json +{ + "results": [ + /* SearchResult[] */ + ], + "query": "string", + "count": 5, + "execution_time_ms": 42.0 +} +``` + +**Errors:** 400 (missing query), 503 (llama-server or Qdrant unavailable) + +--- + +### `POST /v0/search/related` + +Find files related to a given file using vector similarity. + +**Request body** + +```json +{ + "file_path": "string (required) — absolute path", + "folder_path": "string | null", + "limit": 10, + "filters": [ + /* MetadataFilter[] */ + ] +} +``` + +**Response 200** + +```json +{ + "results": [{ "path": "string", "score": 0.95 }], + "file_path": "string", + "count": 5, + "execution_time_ms": 12.0 +} +``` + +**Errors:** 400, 404 (no indexed chunks for the file), 503 + +--- + +## Folders + +### `GET /v0/folder` + +- With `?path=`: returns a single `FolderEntry` (404 if not registered) +- Without `path`: returns `{ "folders": [ FolderEntry[] ] }` + +--- + +### `POST /v0/folder` + +Register a folder for indexing. Starts watching and scanning immediately. + +**Request body** + +```json +{ + "path": "string (required) — absolute path", + "include_patterns": ["**/*.md"], + "exclude_patterns": ["**/node_modules/**"], + "recursive": true +} +``` + +**Response 201** — `FolderEntry` + +**Errors:** 400 (invalid), 409 (already registered) + +--- + +### `PATCH /v0/folder` + +Update folder configuration. Only provided fields are changed. + +**Request body** + +```json +{ + "path": "string (required)", + "include_patterns": ["**/*.md"], + "exclude_patterns": ["**/node_modules/**"], + "recursive": false +} +``` + +**Response 200** — updated `FolderEntry` + +**Errors:** 400, 404 + +--- + +### `DELETE /v0/folder` + +Unregister a folder and remove all its indexed data. + +**Request body** + +```json +{ "path": "string (required)" } +``` + +**Response 200** — deletion summary object + +**Errors:** 400, 404 + +--- + +### `POST /v0/folder/pause` + +Stop file watching for a folder without removing it. + +**Request body** + +```json +{ "path": "string (required)" } +``` + +**Response 200** + +```json +{ "status": "paused", "path": "string" } +``` + +**Errors:** 400, 404 + +--- + +### `POST /v0/folder/resume` + +Resume file watching and trigger a rescan. + +**Request body** + +```json +{ "path": "string (required)" } +``` + +**Response 202** + +```json +{ "status": "scanning", "path": "string" } +``` + +**Errors:** 400, 404 + +--- + +### `POST /v0/scan` + +Manually trigger a rescan of a registered folder. + +**Request body** + +```json +{ + "path": "string (required)", + "force": false +} +``` + +`force: true` re-indexes all files even if unchanged. + +**Response 202** + +```json +{ "status": "started", "path": "string" } +``` + +**Errors:** 400, 404 + +--- + +## Files & Documents + +### `GET /v0/folder/files` + +List indexed files with optional filtering and pagination. + +**Query parameters** + +| Param | Type | Description | +| -------------- | --------------------------------- | ------------------------------- | +| `folder_path` | string | Filter by folder | +| `title` | string | Substring match on title | +| `file_path` | string | Exact file path match | +| `mtime_after` | number | Unix timestamp lower bound | +| `mtime_before` | number | Unix timestamp upper bound | +| `offset` | integer (default 0) | Pagination offset | +| `limit` | integer | Max results (omit for no limit) | +| `order_by` | `mtime` \| `updated_at` (default) | Sort order | + +**Response 200** + +```json +{ + "files": [ + /* FileEntry[] */ + ], + "total": 99 +} +``` + +--- + +### `GET /v0/folder/documents` + +Fetch all indexed chunks for a specific file, sorted by chunk index. + +**Query parameters** + +| Param | Required | Description | +| ------------- | -------- | -------------------------- | +| `path` | yes | Absolute file path | +| `folder_path` | no | Scope to a specific folder | + +**Response 200** + +```json +{ + "documents": [ + /* DocumentChunk[] */ + ] +} +``` + +**Errors:** 400, 503 + +--- + +## Utilities + +### `POST /v0/parse-doc` + +Parse a file and return its extracted text content. + +**Request body** + +```json +{ "path": "string (required) — absolute file path" } +``` + +**Response 200** — parsed content object (shape varies by file type) + +**Errors:** + +| Code | Meaning | +| ---- | --------------------- | +| 400 | Invalid input | +| 403 | File not readable | +| 404 | File not found | +| 415 | Unsupported file type | +| 422 | Parse failed | +| 500 | Internal error | + +--- + +### `POST /v0/rebuild-metadata` + +Rebuild the manifest by re-syncing metadata from Qdrant. Use when manifest is out of sync. + +**Response 200** — `{ "elapsed_ms": 123, ...stats }` + +**Errors:** 409 (rebuild already in progress), 503 + +--- + +### `POST /v0/llama-server/restart` + +Restart the llama-server sidecar, optionally changing the batch size preset. + +**Request body** + +```json +{ "batch_size_preset": "default" } +``` + +**Response 200** + +```json +{ + "restarted": true, + "batch_size_preset": "default", + "status": "running" +} +``` + +**Errors:** 400 + +--- + +### `POST /v1/embeddings` + +Generate embeddings. OpenAI-compatible interface, proxied to llama-server. + +**Request body** + +```json +{ + "model": "nomic-embed-text-v1.5", + "input": "string or string[]" +} +``` + +`model` is optional but must match the configured embedding model if provided. + +**Response 200** — standard OpenAI embeddings response + +**Errors:** 400, 503 + +--- + +## Schemas + +### MetadataFilter + +Range filter on a metadata field. + +```json +{ + "field": "mtime", + "gt": 1700000000, + "gte": 1700000000, + "lt": 1800000000, + "lte": 1800000000 +} +``` + +- `field` can be `mtime`, `ctime`, or any metadata key +- Bare field names (not `mtime`/`ctime` and not prefixed with `metadata.`) are automatically prefixed with `metadata.` +- At least one of `gt`, `gte`, `lt`, `lte` must be present + +--- + +### SearchResult + +```json +{ + "path": "string", + "score": 0.95, + "title": "string | null", + "mtime": 1700000000, + "ctime": 1700000000, + "file_name": "string | null", + "chunk_index": 0, + "total_chunks": 5, + "chunk_text": "string | null", + "metadata": {}, + "embedding_model": "string | null", + "tags": ["string"], + "extension": ".md", + "created_at": "string | null", + "nchars": 1024, + "folder_path": "string | null" +} +``` + +--- + +### FileEntry + +```json +{ + "path": "string", + "title": "string | null", + "mtime": 1700000000, + "updated_at": "ISO8601 string", + "folder_path": "string | null", + "total_chunks": 5 +} +``` + +--- + +### DocumentChunk + +```json +{ + "id": "string", + "path": "string | null", + "title": "string | null", + "chunk_index": 0, + "chunk_text": "string | null", + "metadata": {}, + "embedding_model": "string | null", + "ctime": 1700000000, + "mtime": 1700000000, + "tags": ["string"], + "extension": ".md", + "created_at": "ISO8601 string | null", + "nchars": 1024, + "folder_path": "string | null" +} +``` + +--- + +### FolderEntry + +Shape varies — includes at minimum: + +```json +{ + "path": "string", + "include_patterns": ["**/*.md"], + "exclude_patterns": [], + "recursive": true +} +``` + +Plus live stats fields populated by the folder manager. diff --git a/src/commands/index.ts b/src/commands/index.ts index e3edf317..d9ab6b3b 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -23,6 +23,7 @@ import { YoutubeTranscriptModal } from "@/components/modals/YoutubeTranscriptMod import { checkIsPlusUser } from "@/plusUtils"; // Debug modals removed with search v3 import CopilotPlugin from "@/main"; +import { shouldUseMiyo } from "@/miyo/miyoUtils"; import { getAllQAMarkdownContent } from "@/search/searchUtils"; import { CopilotSettings } from "@/settings/model"; import { NoteSelectedTextContext, WebSelectedTextContext } from "@/types/message"; @@ -179,10 +180,16 @@ export function registerCommands( addCommand(plugin, COMMAND_IDS.CLEAR_LOCAL_COPILOT_INDEX, async () => { const { getSettings } = await import("@/settings/model"); - const isMiyoEnabled = getSettings().enableMiyo; - const clearMessage = isMiyoEnabled - ? "This will permanently delete all indexes of this vault from Miyo. This action cannot be undone.\n\nAre you sure you want to proceed?" - : "This will permanently delete all document indexes in Copilot. This action cannot be undone.\n\nAre you sure you want to proceed?"; + const settings = getSettings(); + const isMiyoEnabled = shouldUseMiyo(settings); + if (isMiyoEnabled) { + new Notice( + "Miyo folders are managed in Miyo. Remove the folder there if you want to clear it." + ); + return; + } + const clearMessage = + "This will permanently delete all document indexes in Copilot. This action cannot be undone.\n\nAre you sure you want to proceed?"; const confirmed = await new Promise((resolve) => { new ConfirmModal( plugin.app, @@ -207,6 +214,13 @@ export function registerCommands( addCommand(plugin, COMMAND_IDS.GARBAGE_COLLECT_COPILOT_INDEX, async () => { try { + const { getSettings } = await import("@/settings/model"); + if (shouldUseMiyo(getSettings())) { + new Notice( + "Miyo manages file cleanup automatically. Run Index (refresh) vault to trigger a scan if needed." + ); + return; + } const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; const removedCount = await VectorStoreManager.getInstance().garbageCollectVectorStore(); new Notice(`Garbage collection completed. Removed ${removedCount} stale documents.`); @@ -229,7 +243,11 @@ export function registerCommands( const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(false, { userInitiated: true, }); - new Notice(`Semantic search index refreshed with ${count} documents.`); + if (shouldUseMiyo(settings)) { + new Notice("Miyo folder index refresh started. Open the Miyo app to check details."); + } else { + new Notice(`Semantic search index refreshed with ${count} documents.`); + } } else { // V3 search builds indexes on demand new Notice("Lexical search builds indexes on demand. No manual indexing required."); @@ -263,7 +281,11 @@ export function registerCommands( const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(true, { userInitiated: true, }); - new Notice(`Semantic search index rebuilt with ${count} documents.`); + if (shouldUseMiyo(settings)) { + new Notice("Miyo folder index refresh started. Open the Miyo app to check details."); + } else { + new Notice(`Semantic search index rebuilt with ${count} documents.`); + } } else { // V3 search builds indexes on demand new Notice("Lexical search builds indexes on demand. No manual indexing required."); diff --git a/src/components/chat-components/ChatControls.tsx b/src/components/chat-components/ChatControls.tsx index f6f064e3..c6f263cb 100644 --- a/src/components/chat-components/ChatControls.tsx +++ b/src/components/chat-components/ChatControls.tsx @@ -8,6 +8,7 @@ import { SettingSwitch } from "@/components/ui/setting-switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { PLUS_UTM_MEDIUMS } from "@/constants"; import { logError } from "@/logger"; +import { shouldUseMiyo } from "@/miyo/miyoUtils"; import { navigateToPlusPage, useIsPlusUser } from "@/plusUtils"; import { updateSetting, useSettingsValue } from "@/settings/model"; import { Docs4LLMParser } from "@/tools/FileParserManager"; @@ -44,9 +45,14 @@ export async function refreshVaultIndex() { if (settings.enableSemanticSearchV3) { // Use VectorStoreManager for semantic search indexing const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; - await VectorStoreManager.getInstance().indexVaultToVectorStore(false, { + const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(false, { userInitiated: true, }); + if (shouldUseMiyo(settings)) { + new Notice("Miyo folder index refresh started. Open the Miyo app to check details."); + } else { + new Notice(`Semantic search index refreshed with ${count} documents.`); + } } else { // V3 search builds indexes on demand new Notice("Lexical search builds indexes on demand. No manual indexing required."); @@ -65,7 +71,14 @@ export async function forceReindexVault() { if (settings.enableSemanticSearchV3) { // Use VectorStoreManager for semantic search indexing const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; - await VectorStoreManager.getInstance().indexVaultToVectorStore(true, { userInitiated: true }); + const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(true, { + userInitiated: true, + }); + if (shouldUseMiyo(settings)) { + new Notice("Miyo folder index refresh started. Open the Miyo app to check details."); + } else { + new Notice(`Semantic search index rebuilt with ${count} documents.`); + } } else { // V3 search builds indexes on demand new Notice("Lexical search builds indexes on demand. No manual indexing required."); diff --git a/src/miyo/MiyoClient.test.ts b/src/miyo/MiyoClient.test.ts index 342d0ce2..96da7852 100644 --- a/src/miyo/MiyoClient.test.ts +++ b/src/miyo/MiyoClient.test.ts @@ -1,3 +1,4 @@ +import { getDecryptedKey } from "@/encryptionService"; import { logInfo } from "@/logger"; import { MiyoClient } from "@/miyo/MiyoClient"; import { MiyoServiceDiscovery } from "@/miyo/MiyoServiceDiscovery"; @@ -12,6 +13,10 @@ jest.mock("@/settings/model", () => ({ getSettings: jest.fn(), })); +jest.mock("@/encryptionService", () => ({ + getDecryptedKey: jest.fn(async (value: string) => value), +})); + const mockResolveBaseUrl = jest.fn(); jest.mock("@/miyo/MiyoServiceDiscovery", () => ({ @@ -28,11 +33,12 @@ jest.mock("@/logger", () => ({ logError: jest.fn(), })); -describe("MiyoClient.parseDoc", () => { +describe("MiyoClient", () => { const mockedRequestUrl = requestUrl as jest.MockedFunction; const mockedGetSettings = getSettings as jest.MockedFunction; const mockedGetInstance = MiyoServiceDiscovery.getInstance as unknown as jest.Mock; const mockedLogInfo = logInfo as jest.MockedFunction; + const mockedGetDecryptedKey = getDecryptedKey as jest.MockedFunction; beforeEach(() => { jest.clearAllMocks(); @@ -40,6 +46,7 @@ describe("MiyoClient.parseDoc", () => { plusLicenseKey: "plus-test-license", debug: false, } as any); + mockedGetDecryptedKey.mockResolvedValue("plus-test-license"); mockResolveBaseUrl.mockResolvedValue("http://127.0.0.1:8742"); mockedGetInstance.mockReturnValue({ resolveBaseUrl: mockResolveBaseUrl, @@ -90,17 +97,86 @@ describe("MiyoClient.parseDoc", () => { ); }); - it("throws when /v0/parse-doc returns an error status", async () => { + it("sends folder_path in /v0/search requests", async () => { mockedRequestUrl.mockResolvedValue({ - status: 500, - text: "internal server error", - json: { detail: "fail" }, + status: 200, + json: { results: [] }, + text: "", + } as any); + + const client = new MiyoClient(); + await client.search("http://127.0.0.1:8742", "/vault", "project notes", 10, [ + { field: "mtime", gte: 1, lte: 2 }, + ]); + + expect(mockedRequestUrl).toHaveBeenCalledWith( + expect.objectContaining({ + url: "http://127.0.0.1:8742/v0/search", + method: "POST", + body: JSON.stringify({ + query: "project notes", + folder_path: "/vault", + limit: 10, + filters: [{ field: "mtime", gte: 1, lte: 2 }], + }), + }) + ); + }); + + it("requests folder scans through /v0/scan", async () => { + mockedRequestUrl.mockResolvedValue({ + status: 202, + json: { status: "started", path: "/vault" }, + text: "", + } as any); + + const client = new MiyoClient(); + const result = await client.scanFolder("http://127.0.0.1:8742", "/vault", true); + + expect(result).toEqual({ status: "started", path: "/vault" }); + expect(mockedRequestUrl).toHaveBeenCalledWith( + expect.objectContaining({ + url: "http://127.0.0.1:8742/v0/scan", + method: "POST", + body: JSON.stringify({ path: "/vault", force: true }), + }) + ); + }); + + it("lists indexed files from /v0/folder/files with folder_path query params", async () => { + mockedRequestUrl.mockResolvedValue({ + status: 200, + json: { files: [], total: 0 }, + text: "", + } as any); + + const client = new MiyoClient(); + await client.listFolderFiles("http://127.0.0.1:8742", { + folderPath: "/vault", + offset: 10, + limit: 25, + orderBy: "mtime", + }); + + expect(mockedRequestUrl).toHaveBeenCalledWith( + expect.objectContaining({ + url: "http://127.0.0.1:8742/v0/folder/files?folder_path=%2Fvault&offset=10&limit=25&order_by=mtime", + method: "GET", + }) + ); + }); + + it("throws detailed errors when a request fails", async () => { + mockedRequestUrl.mockResolvedValue({ + status: 404, + text: "not found", + json: { detail: "folder not registered" }, } as any); const client = new MiyoClient(); - await expect(client.parseDoc("http://127.0.0.1:8742", "/tmp/sample.pdf")).rejects.toThrow( - "Miyo request failed with status 500" + await expect(client.getFolder("http://127.0.0.1:8742", "/vault")).rejects.toThrow( + "Miyo request failed with status 404: folder not registered" ); }); }); diff --git a/src/miyo/MiyoClient.ts b/src/miyo/MiyoClient.ts index 7d4c6123..c8d7179b 100644 --- a/src/miyo/MiyoClient.ts +++ b/src/miyo/MiyoClient.ts @@ -1,44 +1,47 @@ import { getDecryptedKey } from "@/encryptionService"; import { logError, logInfo, logWarn } from "@/logger"; +import { MiyoServiceDiscovery } from "@/miyo/MiyoServiceDiscovery"; import { getSettings } from "@/settings/model"; import { err2String } from "@/utils"; import { requestUrl } from "obsidian"; -import { MiyoServiceDiscovery } from "@/miyo/MiyoServiceDiscovery"; /** - * Request payload for upserting documents into Miyo. + * Indexed file entry returned by Miyo. */ -export interface MiyoUpsertDocument { - id: string; +export interface MiyoIndexedFileEntry { path: string; - title: string; - content: string; - created_at: number; - ctime: number; + title?: string | null; mtime: number; - tags: string[]; - extension: string; - nchars: number; - metadata: Record; + updated_at?: string; + folder_path?: string | null; + total_chunks?: number; } /** - * Response for index file listing. + * Response for indexed file listing. */ export interface MiyoIndexedFilesResponse { - files: Array<{ path: string; mtime: number }>; + files: MiyoIndexedFileEntry[]; total: number; } /** - * Response for index statistics. + * Folder entry returned by Miyo. */ -export interface MiyoIndexStatsResponse { - total_chunks: number; - total_files: number; - latest_mtime: number; - embedding_model?: string; - embedding_dim?: number; +export interface MiyoFolderEntry { + path: string; + include_patterns?: string[]; + exclude_patterns?: string[]; + recursive?: boolean; + [key: string]: unknown; +} + +/** + * Response for scan requests. + */ +export interface MiyoScanResponse { + status?: string; + path?: string; } /** @@ -48,17 +51,18 @@ export interface MiyoDocumentsResponse { documents: Array<{ id: string; path: string; - title?: string; + title?: string | null; chunk_index?: number; - chunk_text?: string; + chunk_text?: string | null; metadata?: Record; - embedding_model?: string; + embedding_model?: string | null; ctime?: number; mtime?: number; tags?: string[]; extension?: string; - created_at?: number; + created_at?: string | number | null; nchars?: number; + folder_path?: string | null; }>; } @@ -69,17 +73,18 @@ export interface MiyoSearchResult { id: string; score: number; path: string; - title?: string; + title?: string | null; chunk_index?: number; - chunk_text: string; + chunk_text?: string | null; metadata?: Record; - embedding_model?: string; + embedding_model?: string | null; ctime?: number; mtime?: number; tags?: string[]; extension?: string; - created_at?: number; + created_at?: string | number | null; nchars?: number; + folder_path?: string | null; } /** @@ -179,113 +184,103 @@ export class MiyoClient { } /** - * Upsert a batch of documents. + * Fetch the Miyo folder entry for a registered folder path. * * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. - * @param documents - Documents to upsert. - * @returns Number of documents upserted. + * @param folderPath - Absolute folder path registered in Miyo. + * @returns Folder entry. */ - public async upsertDocuments( + public async getFolder(baseUrl: string, folderPath: string): Promise { + return this.requestJson(baseUrl, "/v0/folder", { + method: "GET", + query: { path: folderPath }, + }); + } + + /** + * Trigger a Miyo folder scan. + * + * @param baseUrl - Miyo base URL. + * @param folderPath - Absolute folder path registered in Miyo. + * @param force - Whether to force a full re-scan. + * @returns Scan response. + */ + public async scanFolder( baseUrl: string, - vault: string, - documents: MiyoUpsertDocument[] - ): Promise { - const payload = { vault, documents }; - const response = await this.requestJson<{ upserted: number }>(baseUrl, "/v0/index/upsert", { + folderPath: string, + force = false + ): Promise { + return this.requestJson(baseUrl, "/v0/scan", { method: "POST", - body: payload, + body: { + path: folderPath, + force, + }, }); - return response?.upserted ?? 0; } /** - * Delete documents by file path. + * List indexed files for a folder with pagination and filters. * * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. - * @param path - File path to delete. - * @returns Count of documents deleted. - */ - public async deleteByPath(baseUrl: string, vault: string, path: string): Promise { - const payload = { vault, path }; - const response = await this.requestJson<{ deleted?: number }>(baseUrl, "/v0/index/by_path", { - method: "DELETE", - body: payload, - }); - return response?.deleted ?? 0; - } - - /** - * Clear all indexed documents for a source. - * - * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. - */ - public async clearIndex(baseUrl: string, vault: string): Promise { - const payload = { vault }; - await this.requestJson(baseUrl, "/v0/index/clear", { method: "POST", body: payload }); - } - - /** - * List indexed files with pagination. - * - * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. - * @param offset - Offset for pagination. - * @param limit - Page size. + * @param options - Folder file list options. * @returns Indexed files response. */ - public async listFiles( + public async listFolderFiles( baseUrl: string, - vault: string, - offset: number, - limit: number + options: { + folderPath: string; + title?: string; + filePath?: string; + mtimeAfter?: number; + mtimeBefore?: number; + offset?: number; + limit?: number; + orderBy?: "mtime" | "updated_at"; + } ): Promise { - return this.requestJson(baseUrl, "/v0/index/files", { + return this.requestJson(baseUrl, "/v0/folder/files", { method: "GET", - query: { vault, offset, limit }, + query: { + folder_path: options.folderPath, + title: options.title, + file_path: options.filePath, + mtime_after: options.mtimeAfter, + mtime_before: options.mtimeBefore, + offset: options.offset, + limit: options.limit, + order_by: options.orderBy, + }, }); } /** - * Fetch index statistics for a source. + * Fetch all indexed chunks for a file path. * * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. - * @returns Index stats response. - */ - public async getStats(baseUrl: string, vault: string): Promise { - return this.requestJson(baseUrl, "/v0/index/stats", { - method: "GET", - query: { vault }, - }); - } - - /** - * Fetch all documents for a given path. - * - * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. - * @param path - File path to look up. + * @param folderPath - Folder path to scope the document lookup. + * @param path - Absolute file path to look up. * @returns Documents response. */ public async getDocumentsByPath( baseUrl: string, - vault: string, + folderPath: string, path: string ): Promise { - return this.requestJson(baseUrl, "/v0/index/documents", { + return this.requestJson(baseUrl, "/v0/folder/documents", { method: "GET", - query: { vault, path }, + query: { + path, + folder_path: folderPath, + }, }); } /** - * Execute a hybrid search query. + * Execute a hybrid search query scoped to a folder path. * * @param baseUrl - Miyo base URL. - * @param vault - Vault identifier sent to Miyo. + * @param folderPath - Folder path sent to Miyo. * @param query - User query. * @param limit - Maximum number of results. * @param filters - Optional search filters. @@ -293,14 +288,14 @@ export class MiyoClient { */ public async search( baseUrl: string, - vault: string, + folderPath: string, query: string, limit: number, filters?: MiyoSearchFilter[] ): Promise { const payload = { query, - vault, + folder_path: folderPath, limit, ...(filters && filters.length > 0 ? { filters } : {}), }; @@ -317,28 +312,25 @@ export class MiyoClient { * Execute related-notes search for a source note path. * * @param baseUrl - Miyo base URL. - * @param filePath - Source note path to find related notes for. - * @param options - Optional source id, result limit, and filters. + * @param filePath - Absolute source note path to find related notes for. + * @param options - Optional folder path, result limit, and filters. * @returns Search response in the same shape as /v0/search. */ public async searchRelated( baseUrl: string, filePath: string, options?: { - vault?: string; + folderPath?: string; limit?: number; filters?: MiyoSearchFilter[]; } ): Promise { const payload = { file_path: filePath, - ...(options?.vault ? { vault: options.vault } : {}), + ...(options?.folderPath ? { folder_path: options.folderPath } : {}), ...(typeof options?.limit === "number" ? { limit: options.limit } : {}), ...(options?.filters && options.filters.length > 0 ? { filters: options.filters } : {}), }; - if (getSettings().debug) { - logInfo("Miyo related search request:", { baseUrl, payload }); - } return this.requestJson(baseUrl, "/v0/search/related", { method: "POST", body: payload, @@ -415,9 +407,6 @@ export class MiyoClient { ...(getSettings().debug && options.method === "POST" ? { postBody: options.body } : {}), }); - // TODO: Add a configurable timeout for large file parsing (e.g. big PDFs). - // Obsidian's requestUrl does not expose a timeout option, so consider using - // AbortController or a wrapper with Promise.race to prevent indefinite hangs. const response = await requestUrl({ url: url.toString(), method: options.method, @@ -428,9 +417,17 @@ export class MiyoClient { }); if (response.status >= 400) { - const errorText = response.text ? response.text : ""; + const errorPayload = this.parseResponseJson<{ detail?: string }>( + response.json, + response.text + ); + const errorText = errorPayload?.detail || response.text || ""; logWarn(`Miyo request failed (${response.status}): ${errorText}`); - throw new Error(`Miyo request failed with status ${response.status}`); + throw new Error( + errorText + ? `Miyo request failed with status ${response.status}: ${errorText}` + : `Miyo request failed with status ${response.status}` + ); } const parsed = this.parseResponseJson(response.json, response.text); diff --git a/src/miyo/miyoUtils.ts b/src/miyo/miyoUtils.ts index dce29aad..4f6d13c0 100644 --- a/src/miyo/miyoUtils.ts +++ b/src/miyo/miyoUtils.ts @@ -1,5 +1,5 @@ import { isSelfHostAccessValid } from "@/plusUtils"; -import { CopilotSettings, getSettings } from "@/settings/model"; +import { CopilotSettings } from "@/settings/model"; import { App, FileSystemAdapter, Platform } from "obsidian"; /** @@ -36,21 +36,12 @@ export function shouldUseMiyo(settings: CopilotSettings): boolean { } /** - * Resolve the vault identifier sent to Miyo. - * - * Uses the user-configured remote vault path only when a remote server URL is also - * configured, otherwise falls back to the vault filesystem path or vault name. + * Resolve the current vault folder path sent to Miyo as `folder_path`. * * @param app - Obsidian application instance. - * @returns Remote vault path when a remote server is configured, vault folder path when available, otherwise vault name. + * @returns Current vault filesystem path when available, otherwise vault name. */ -export function getMiyoVault(app: App): string { - const settings = getSettings(); - const remoteVaultPath = (settings.miyoRemoteVaultPath || "").trim(); - const serverUrl = (settings.miyoServerUrl || "").trim(); - if (remoteVaultPath && serverUrl) { - return remoteVaultPath; - } +export function getMiyoFolderPath(app: App): string { const vaultPath = getVaultBasePath(app); if (vaultPath) { return vaultPath; @@ -59,22 +50,48 @@ export function getMiyoVault(app: App): string { } /** - * Resolve the vault identifier that would apply given a vault path override, used for UI preview. + * Resolve an absolute filesystem path for a vault file so it can be sent to Miyo. * * @param app - Obsidian application instance. - * @param vaultPathOverride - The vault path to use (empty string = auto-detect). - * @returns Overridden vault path, or auto-detected vault path/name. + * @param vaultRelativePath - Vault-relative note path. + * @returns Absolute file path when available, otherwise the original path. */ -export function resolveMiyoVault(app: App, vaultNameOverride: string): string { - const trimmed = vaultNameOverride.trim(); - if (trimmed) { - return trimmed; +export function getMiyoAbsolutePath(app: App, vaultRelativePath: string): string { + const adapter = app.vault.adapter; + if (adapter instanceof FileSystemAdapter) { + return adapter.getFullPath(vaultRelativePath); } + + const adapterAny = adapter as unknown as { getFullPath?: (normalizedPath: string) => string }; + if (typeof adapterAny.getFullPath === "function") { + return adapterAny.getFullPath(vaultRelativePath); + } + + return vaultRelativePath; +} + +/** + * Convert a Miyo file path back to a vault-relative path when it belongs to the current vault. + * + * @param app - Obsidian application instance. + * @param miyoPath - Path returned by Miyo. + * @returns Vault-relative path when the file is inside the current vault, otherwise the original path. + */ +export function getVaultRelativeMiyoPath(app: App, miyoPath: string): string { const vaultPath = getVaultBasePath(app); - if (vaultPath) { - return vaultPath; + if (!vaultPath) { + return miyoPath; } - return app.vault.getName(); + + const normalizedVaultPath = normalizeFilesystemPath(vaultPath); + const normalizedMiyoPath = normalizeFilesystemPath(miyoPath); + const prefix = `${normalizedVaultPath}/`; + + if (normalizedMiyoPath.startsWith(prefix)) { + return normalizedMiyoPath.slice(prefix.length); + } + + return miyoPath; } /** @@ -83,7 +100,7 @@ export function resolveMiyoVault(app: App, vaultNameOverride: string): string { * @param app - Obsidian application instance. * @returns Vault base path or undefined when unavailable. */ -function getVaultBasePath(app: App): string | undefined { +export function getVaultBasePath(app: App): string | undefined { const adapter = app.vault.adapter; if (adapter instanceof FileSystemAdapter) { return adapter.getBasePath(); @@ -98,3 +115,13 @@ function getVaultBasePath(app: App): string | undefined { } return undefined; } + +/** + * Normalize a filesystem path for prefix comparisons across platforms. + * + * @param path - Filesystem path. + * @returns Normalized path with forward slashes and no trailing slash. + */ +function normalizeFilesystemPath(path: string): string { + return path.replace(/\\/g, "/").replace(/\/+$/, ""); +} diff --git a/src/search/findRelevantNotes.test.ts b/src/search/findRelevantNotes.test.ts index 5187b9da..a6807716 100644 --- a/src/search/findRelevantNotes.test.ts +++ b/src/search/findRelevantNotes.test.ts @@ -2,7 +2,12 @@ import { TFile } from "obsidian"; import { getBacklinkedNotes, getLinkedNotes } from "@/noteUtils"; import { findRelevantNotes } from "@/search/findRelevantNotes"; import { MiyoClient } from "@/miyo/MiyoClient"; -import { getMiyoVault, shouldUseMiyo } from "@/miyo/miyoUtils"; +import { + getMiyoAbsolutePath, + getMiyoFolderPath, + getVaultRelativeMiyoPath, + shouldUseMiyo, +} from "@/miyo/miyoUtils"; import { getSettings } from "@/settings/model"; import VectorStoreManager from "@/search/vectorStoreManager"; @@ -47,7 +52,9 @@ jest.mock("@/miyo/MiyoClient", () => ({ })); jest.mock("@/miyo/miyoUtils", () => ({ - getMiyoVault: jest.fn(), + getMiyoFolderPath: jest.fn(), + getMiyoAbsolutePath: jest.fn((_: unknown, path: string) => `/vault/${path}`), + getVaultRelativeMiyoPath: jest.fn((_: unknown, path: string) => path.replace("/vault/", "")), getMiyoCustomUrl: jest.fn().mockReturnValue(""), shouldUseMiyo: jest.fn(), })); @@ -76,7 +83,15 @@ describe("findRelevantNotes", () => { const mockedGetBacklinkedNotes = getBacklinkedNotes as jest.MockedFunction< typeof getBacklinkedNotes >; - const mockedGetMiyoSourceId = getMiyoVault as jest.MockedFunction; + const mockedGetMiyoFolderPath = getMiyoFolderPath as jest.MockedFunction< + typeof getMiyoFolderPath + >; + const mockedGetMiyoAbsolutePath = getMiyoAbsolutePath as jest.MockedFunction< + typeof getMiyoAbsolutePath + >; + const mockedGetVaultRelativeMiyoPath = getVaultRelativeMiyoPath as jest.MockedFunction< + typeof getVaultRelativeMiyoPath + >; const mockedVectorStoreManager = VectorStoreManager as unknown as { getInstance: () => { getDocumentsByPath: jest.Mock; @@ -98,7 +113,11 @@ describe("findRelevantNotes", () => { } as any); mockedGetLinkedNotes.mockReturnValue([]); mockedGetBacklinkedNotes.mockReturnValue([]); - mockedGetMiyoSourceId.mockReturnValue("test-source"); + mockedGetMiyoFolderPath.mockReturnValue("/vault"); + mockedGetMiyoAbsolutePath.mockImplementation((_: unknown, path: string) => `/vault/${path}`); + mockedGetVaultRelativeMiyoPath.mockImplementation((_: unknown, path: string) => + path.replace("/vault/", "") + ); const source = createMarkdownFile("source.md"); const first = createMarkdownFile("first.md"); @@ -198,10 +217,10 @@ describe("findRelevantNotes", () => { mockResolveBaseUrl.mockResolvedValue("http://127.0.0.1:8742"); mockSearchRelated.mockResolvedValue({ results: [ - { id: "self", path: "source.md", score: 0.99, chunk_text: "self" }, - { id: "a-1", path: "alpha.md", score: 0.45, chunk_text: "alpha1" }, - { id: "b-1", path: "beta.md", score: 0.88, chunk_text: "beta" }, - { id: "a-2", path: "alpha.md", score: 0.6, chunk_text: "alpha2" }, + { id: "self", path: "/vault/source.md", score: 0.99, chunk_text: "self" }, + { id: "a-1", path: "/vault/alpha.md", score: 0.45, chunk_text: "alpha1" }, + { id: "b-1", path: "/vault/beta.md", score: 0.88, chunk_text: "beta" }, + { id: "a-2", path: "/vault/alpha.md", score: 0.6, chunk_text: "alpha2" }, ], }); @@ -214,8 +233,8 @@ describe("findRelevantNotes", () => { expect(mockGetDb).not.toHaveBeenCalled(); expect(mockGetDocumentsByPath).not.toHaveBeenCalled(); expect(mockSearchRelated).toHaveBeenCalledTimes(1); - expect(mockSearchRelated).toHaveBeenCalledWith("http://127.0.0.1:8742", "source.md", { - vault: "test-source", + expect(mockSearchRelated).toHaveBeenCalledWith("http://127.0.0.1:8742", "/vault/source.md", { + folderPath: "/vault", limit: 20, }); }); @@ -235,8 +254,8 @@ describe("findRelevantNotes", () => { mockResolveBaseUrl.mockResolvedValue("http://127.0.0.1:8742"); mockSearchRelated.mockResolvedValue({ results: [ - { id: "a-1", path: "alpha.md", score: 0.75, chunk_text: "alpha chunk" }, - { id: "self", path: "source.md", score: 0.99, chunk_text: "self" }, + { id: "a-1", path: "/vault/alpha.md", score: 0.75, chunk_text: "alpha chunk" }, + { id: "self", path: "/vault/source.md", score: 0.99, chunk_text: "self" }, ], }); diff --git a/src/search/findRelevantNotes.ts b/src/search/findRelevantNotes.ts index 02f4d8e4..8bac26ea 100644 --- a/src/search/findRelevantNotes.ts +++ b/src/search/findRelevantNotes.ts @@ -1,6 +1,12 @@ import { logInfo, logWarn } from "@/logger"; import { MiyoClient } from "@/miyo/MiyoClient"; -import { getMiyoCustomUrl, getMiyoVault, shouldUseMiyo } from "@/miyo/miyoUtils"; +import { + getMiyoAbsolutePath, + getMiyoCustomUrl, + getMiyoFolderPath, + getVaultRelativeMiyoPath, + shouldUseMiyo, +} from "@/miyo/miyoUtils"; import { getBacklinkedNotes, getLinkedNotes } from "@/noteUtils"; import { DBOperations } from "@/search/dbOperations"; import type { SemanticIndexDocument } from "@/search/indexBackend/SemanticIndexBackend"; @@ -123,24 +129,25 @@ async function calculateSimilarityScoreFromMiyo(filePath: string): Promise(); const results = response.results || []; for (const result of results) { - if (result.path === filePath) { + const relativePath = getVaultRelativeMiyoPath(app, result.path); + if (relativePath === filePath) { continue; } if (typeof result.score !== "number" || Number.isNaN(result.score)) { continue; } - const existing = similarityScoreMap.get(result.path); + const existing = similarityScoreMap.get(relativePath); if (existing === undefined || result.score > existing) { - similarityScoreMap.set(result.path, result.score); + similarityScoreMap.set(relativePath, result.score); } } diff --git a/src/search/indexBackend/MiyoIndexBackend.ts b/src/search/indexBackend/MiyoIndexBackend.ts index 21b3188d..da636f58 100644 --- a/src/search/indexBackend/MiyoIndexBackend.ts +++ b/src/search/indexBackend/MiyoIndexBackend.ts @@ -1,10 +1,13 @@ import { Embeddings } from "@langchain/core/embeddings"; import { App, Notice } from "obsidian"; -import { CustomError } from "@/error"; -import { logError, logInfo, logWarn } from "@/logger"; -import { MiyoClient, MiyoUpsertDocument } from "@/miyo/MiyoClient"; -import { getMiyoCustomUrl, getMiyoVault } from "@/miyo/miyoUtils"; -import { getMatchingPatterns, shouldIndexFile } from "@/search/searchUtils"; +import { logInfo, logWarn } from "@/logger"; +import { MiyoClient, MiyoIndexedFileEntry } from "@/miyo/MiyoClient"; +import { + getMiyoAbsolutePath, + getMiyoCustomUrl, + getMiyoFolderPath, + getVaultRelativeMiyoPath, +} from "@/miyo/miyoUtils"; import type { SemanticIndexBackend, SemanticIndexDocument, @@ -13,10 +16,12 @@ import { getSettings } from "@/settings/model"; /** * Miyo-backed implementation of the semantic index backend. + * + * Miyo owns folder registration, chunking, indexing, and file-change tracking. + * Copilot only requests scans and reads indexed results. */ export class MiyoIndexBackend implements SemanticIndexBackend { private client: MiyoClient; - private filesMissingEmbeddings: Set = new Set(); /** * Create a new Miyo backend tied to the current Obsidian app instance. @@ -28,28 +33,29 @@ export class MiyoIndexBackend implements SemanticIndexBackend { } /** - * Initialize the Miyo backend by ensuring the service is reachable. + * Initialize the Miyo backend by validating connectivity and folder access. * * @param _embeddingInstance - Embeddings instance (unused for Miyo initialization). */ public async initialize(_embeddingInstance: Embeddings | undefined): Promise { try { const baseUrl = await this.getBaseUrl(); - await this.client.getStats(baseUrl, this.getSourceId()); + await this.client.getFolder(baseUrl, this.getFolderPath()); } catch (error) { logWarn(`Miyo backend initialization failed: ${error}`); - new Notice("Failed to initialize Miyo backend. Check Miyo service discovery or URL."); + new Notice( + "Failed to initialize Miyo backend. Check Miyo service discovery or folder setup." + ); } } /** - * Clear all indexed data for the current vault. + * Clearing the index is managed directly in Miyo, not by Copilot. * - * @param _embeddingInstance - Embeddings instance (unused for Miyo clear). + * @param _embeddingInstance - Embeddings instance (unused for Miyo). */ public async clearIndex(_embeddingInstance: Embeddings | undefined): Promise { - const baseUrl = await this.getBaseUrl(); - await this.client.clearIndex(baseUrl, this.getSourceId()); + logWarn("Miyo clearIndex requested from Copilot, but folder lifecycle is managed in Miyo."); } /** @@ -62,96 +68,66 @@ export class MiyoIndexBackend implements SemanticIndexBackend { } /** - * Insert or update a single document. + * Copilot no longer upserts individual Miyo documents. * * @param doc - Document to upsert. - * @returns The document when upsert succeeds. + * @returns Undefined because the write is handled by Miyo scans. */ public async upsert(doc: SemanticIndexDocument): Promise { - const count = await this.upsertBatch([doc]); - return count > 0 ? doc : undefined; + logInfo(`Skipping direct Miyo upsert for ${doc.path}; Miyo manages indexing itself.`); + return undefined; } /** - * Insert or update multiple documents in a batch. + * Copilot no longer batches document upserts into Miyo. * * @param docs - Documents to upsert. - * @returns Number of documents upserted. + * @returns Zero because direct upserts are no longer used. */ public async upsertBatch(docs: SemanticIndexDocument[]): Promise { - if (docs.length === 0) { - return 0; + if (docs.length > 0) { + logInfo(`Skipping direct Miyo batch upsert for ${docs.length} documents.`); } - const baseUrl = await this.getBaseUrl(); - const sourceId = this.getSourceId(); - this.logUpsertBatchRequested(docs, baseUrl, sourceId); - const payloadDocs = docs.map((doc) => this.toMiyoDocument(doc)); - const upserted = await this.client.upsertDocuments(baseUrl, sourceId, payloadDocs); - this.logUpsertBatchResult(docs.length, upserted); - return upserted; + return 0; } /** - * Remove all documents associated with a file path. + * File removal is managed by Miyo's folder watcher and scanner. * * @param path - File path to delete. */ public async removeByPath(path: string): Promise { - const baseUrl = await this.getBaseUrl(); - await this.client.deleteByPath(baseUrl, this.getSourceId(), path); + logInfo(`Skipping direct Miyo delete for ${path}; Miyo manages file lifecycle itself.`); } /** - * Return all indexed file paths. + * Return all indexed file paths for the current vault folder. */ public async getIndexedFiles(): Promise { - const baseUrl = await this.getBaseUrl(); - const sourceId = this.getSourceId(); - const limit = 200; - let offset = 0; - let total: number | null = null; - const paths = new Set(); - - do { - const response = await this.client.listFiles(baseUrl, sourceId, offset, limit); - const files = response.files ?? []; - files.forEach((file) => paths.add(file.path)); - // Read total only from the first response to avoid drift from concurrent indexing. - if (total === null) { - total = response.total ?? files.length; - } - offset += files.length; - if (files.length === 0) { - break; - } - } while (offset < total); - - return Array.from(paths).sort(); + const files = await this.getAllIndexedFiles(); + return Array.from(new Set(files.map((file) => this.toVaultPath(file.path)))).sort(); } /** * Return the latest modified time across indexed files. */ public async getLatestFileMtime(): Promise { - const stats = await this.getStatsSafely(); - return stats?.latest_mtime ?? 0; + const files = await this.getAllIndexedFiles(); + return files.reduce((latest, file) => Math.max(latest, file.mtime ?? 0), 0); } /** * Return true when the index has no documents. */ public async isIndexEmpty(): Promise { - const stats = await this.getStatsSafely(); - if (!stats) { - return true; - } - return (stats.total_chunks ?? 0) === 0; + const files = await this.getAllIndexedFiles(); + return files.length === 0; } /** * Return true when the specified file path has indexed documents. * - * @param path - File path to check. + * @param path - Vault-relative file path to check. */ public async hasIndex(path: string): Promise { const docs = await this.getDocumentsByPath(path); @@ -159,13 +135,18 @@ export class MiyoIndexBackend implements SemanticIndexBackend { } /** - * Return all indexed documents for a given file path. + * Return all indexed documents for a given vault-relative file path. * - * @param path - File path to look up. + * @param path - Vault-relative file path to look up. */ public async getDocumentsByPath(path: string): Promise { const baseUrl = await this.getBaseUrl(); - const response = await this.client.getDocumentsByPath(baseUrl, this.getSourceId(), path); + const absolutePath = getMiyoAbsolutePath(this.app, path); + const response = await this.client.getDocumentsByPath( + baseUrl, + this.getFolderPath(), + absolutePath + ); const docs = response.documents ?? []; return docs.map((doc) => this.fromMiyoDocument(path, doc)); } @@ -174,7 +155,7 @@ export class MiyoIndexBackend implements SemanticIndexBackend { * Detect embedding model changes and indicate whether a rebuild is needed. * * @param _embeddingInstance - Current embeddings instance (unused for Miyo). - * @returns True when a rebuild should occur. + * @returns False because Miyo controls embeddings. */ public async checkAndHandleEmbeddingModelChange( _embeddingInstance?: Embeddings @@ -197,67 +178,38 @@ export class MiyoIndexBackend implements SemanticIndexBackend { } /** - * Remove stale documents that no longer match the vault state. + * Garbage collection is managed directly in Miyo. + * + * @returns Zero because Copilot does not delete Miyo records directly anymore. */ public async garbageCollect(): Promise { - try { - const files = this.app.vault.getMarkdownFiles(); - const filePaths = new Set(files.map((file) => file.path)); - - const { inclusions, exclusions } = getMatchingPatterns(); - const allowedPaths = new Set( - files - .filter((file) => shouldIndexFile(file, inclusions, exclusions)) - .map((file) => file.path) - ); - - const indexedPaths = await this.getIndexedFiles(); - const pathsToRemove = indexedPaths.filter( - (path) => !filePaths.has(path) || !allowedPaths.has(path) - ); - - if (pathsToRemove.length === 0) { - return 0; - } - - logInfo("Miyo index: Paths to remove during garbage collection:", pathsToRemove.join(", ")); - - const baseUrl = await this.getBaseUrl(); - const sourceId = this.getSourceId(); - let deleted = 0; - - for (const path of pathsToRemove) { - deleted += await this.client.deleteByPath(baseUrl, sourceId, path); - } - - return deleted; - } catch (error) { - logError("Error garbage collecting the Miyo index:", error); - throw new CustomError("Failed to garbage collect the Copilot index."); - } + logInfo("Skipping Miyo garbage collection; folder lifecycle is managed in Miyo."); + return 0; } /** - * Mark a file as missing embeddings. + * Track missing embeddings (unused for Miyo). * - * @param path - File path to track. + * @param _path - File path to track. */ - public markFileMissingEmbeddings(path: string): void { - this.filesMissingEmbeddings.add(path); + public markFileMissingEmbeddings(_path: string): void { + return; } /** - * Clear tracked files missing embeddings. + * Clear tracked files missing embeddings (unused for Miyo). */ public clearFilesMissingEmbeddings(): void { - this.filesMissingEmbeddings.clear(); + return; } /** - * Return tracked files missing embeddings. + * Return tracked files missing embeddings (unused for Miyo). + * + * @returns Empty array. */ public getFilesMissingEmbeddings(): string[] { - return Array.from(this.filesMissingEmbeddings); + return []; } /** @@ -283,6 +235,16 @@ export class MiyoIndexBackend implements SemanticIndexBackend { return true; } + /** + * Trigger a folder scan in Miyo. + * + * @param force - Whether to request a forced re-scan. + */ + public async requestIndexRefresh(force = false): Promise { + const baseUrl = await this.getBaseUrl(); + await this.client.scanFolder(baseUrl, this.getFolderPath(), force); + } + /** * Resolve the Miyo base URL using settings and discovery. * @@ -294,90 +256,60 @@ export class MiyoIndexBackend implements SemanticIndexBackend { } /** - * Compute the Miyo source id for the current vault. + * Return the Miyo folder path for the current vault. * - * @returns Source id string. + * @returns Folder path string. */ - private getSourceId(): string { - return getMiyoVault(this.app); + private getFolderPath(): string { + return getMiyoFolderPath(this.app); } /** - * Log details about the pending upsert batch for debugging parity with Orama. + * Fetch every indexed file entry from Miyo for the current folder. * - * @param docs - Documents being upserted. - * @param baseUrl - Miyo base URL. - * @param sourceId - Vault source id. + * @returns Indexed file entries. */ - private logUpsertBatchRequested( - docs: SemanticIndexDocument[], - baseUrl: string, - sourceId: string - ): void { - logInfo(`Miyo upsert batch: ${docs.length} documents`, { baseUrl, sourceId }); - docs.forEach((doc) => { - const chunkId = doc.metadata?.chunkId; - const chunkSuffix = - typeof chunkId === "string" && chunkId.length > 0 ? ` (chunkId: ${chunkId})` : ""; - logInfo(`Miyo upsert document ${doc.id} @ ${doc.path}${chunkSuffix}`); - }); + private async getAllIndexedFiles(): Promise { + const baseUrl = await this.getBaseUrl(); + const limit = 200; + let offset = 0; + let total: number | null = null; + const files: MiyoIndexedFileEntry[] = []; + + do { + const response = await this.client.listFolderFiles(baseUrl, { + folderPath: this.getFolderPath(), + offset, + limit, + }); + const batch = response.files ?? []; + files.push(...batch); + if (total === null) { + total = response.total ?? batch.length; + } + offset += batch.length; + if (batch.length === 0) { + break; + } + } while (offset < total); + + return files; } /** - * Log the result of a Miyo upsert batch. + * Convert a Miyo path to a vault-relative path when possible. * - * @param requested - Number of documents requested for upsert. - * @param upserted - Number of documents reported by Miyo. + * @param path - Miyo path value. + * @returns Vault-relative path when the file is inside the current vault. */ - private logUpsertBatchResult(requested: number, upserted: number): void { - if (upserted === requested) { - logInfo(`Miyo upsert batch completed: ${upserted}/${requested} documents`); - return; - } - logWarn(`Miyo upsert batch returned ${upserted}/${requested} documents`); - } - - /** - * Safely fetch index stats without throwing. - * - * @returns Stats response or null when unavailable. - */ - private async getStatsSafely() { - try { - const baseUrl = await this.getBaseUrl(); - return await this.client.getStats(baseUrl, this.getSourceId()); - } catch (error) { - logError(`Failed to fetch Miyo index stats: ${error}`); - return null; - } - } - - /** - * Convert a Copilot document to a Miyo upsert payload. - * - * @param doc - Copilot document. - * @returns Miyo upsert document payload. - */ - private toMiyoDocument(doc: SemanticIndexDocument): MiyoUpsertDocument { - return { - id: doc.id, - path: doc.path, - title: doc.title, - content: doc.content, - created_at: doc.created_at, - ctime: doc.ctime, - mtime: doc.mtime, - tags: doc.tags, - extension: doc.extension, - nchars: doc.nchars, - metadata: doc.metadata ?? {}, - }; + private toVaultPath(path: string): string { + return getVaultRelativeMiyoPath(this.app, path); } /** * Convert a Miyo document response to a Copilot document shape. * - * @param fallbackPath - Path used when payload omits it. + * @param fallbackPath - Vault-relative fallback path used when payload omits it. * @param doc - Miyo response document. * @returns Copilot document shape. */ @@ -386,23 +318,24 @@ export class MiyoIndexBackend implements SemanticIndexBackend { doc: { id: string; path?: string; - title?: string; + title?: string | null; chunk_index?: number; - chunk_text?: string; + chunk_text?: string | null; metadata?: Record; - embedding_model?: string; + embedding_model?: string | null; ctime?: number; mtime?: number; tags?: string[]; extension?: string; - created_at?: number; + created_at?: string | number | null; nchars?: number; } ): SemanticIndexDocument { + const resolvedPath = doc.path ? this.toVaultPath(doc.path) : fallbackPath; const content = doc.chunk_text ?? ""; const metadata = { ...(doc.metadata ?? {}) }; if (!metadata.chunkId && doc.chunk_index !== undefined) { - metadata.chunkId = `${doc.path || fallbackPath}#${doc.chunk_index}`; + metadata.chunkId = `${resolvedPath}#${doc.chunk_index}`; } if (doc.chunk_index !== undefined && metadata.chunkIndex === undefined) { metadata.chunkIndex = doc.chunk_index; @@ -413,9 +346,9 @@ export class MiyoIndexBackend implements SemanticIndexBackend { title: doc.title ?? "", content, embedding: [], - path: doc.path ?? fallbackPath, + path: resolvedPath, embeddingModel: doc.embedding_model ?? "", - created_at: doc.created_at ?? 0, + created_at: typeof doc.created_at === "number" ? doc.created_at : 0, ctime: doc.ctime ?? 0, mtime: doc.mtime ?? 0, tags: doc.tags ?? [], diff --git a/src/search/indexEventHandler.ts b/src/search/indexEventHandler.ts index 68ab7028..36f4549b 100644 --- a/src/search/indexEventHandler.ts +++ b/src/search/indexEventHandler.ts @@ -32,7 +32,11 @@ export class IndexEventHandler { * @returns {boolean} True when semantic search indexing should run. */ private shouldHandleEvents(): boolean { - return getSettings().enableSemanticSearchV3; + if (!getSettings().enableSemanticSearchV3) { + return false; + } + + return !(this.indexBackend.isRemoteBackend() && !this.indexBackend.requiresEmbeddings()); } /** diff --git a/src/search/miyo/MiyoSemanticRetriever.test.ts b/src/search/miyo/MiyoSemanticRetriever.test.ts index e2ca78f7..c270456d 100644 --- a/src/search/miyo/MiyoSemanticRetriever.test.ts +++ b/src/search/miyo/MiyoSemanticRetriever.test.ts @@ -1,4 +1,4 @@ -import { getMiyoVault } from "@/miyo/miyoUtils"; +import { getMiyoFolderPath } from "@/miyo/miyoUtils"; import { MiyoSemanticRetriever } from "@/search/miyo/MiyoSemanticRetriever"; import { getSettings } from "@/settings/model"; import { RETURN_ALL_LIMIT } from "@/search/v3/SearchCore"; @@ -12,7 +12,8 @@ jest.mock("@/settings/model", () => ({ getSettings: jest.fn(), })); jest.mock("@/miyo/miyoUtils", () => ({ - getMiyoVault: jest.fn(), + getMiyoFolderPath: jest.fn(), + getVaultRelativeMiyoPath: jest.fn((_: unknown, path: string) => path.replace("/vault/", "")), getMiyoCustomUrl: jest.fn().mockReturnValue(""), })); jest.mock("@/miyo/MiyoClient", () => ({ @@ -47,7 +48,7 @@ describe("MiyoSemanticRetriever", () => { miyoServerUrl: "http://miyo.local", debug: false, }); - (getMiyoVault as jest.Mock).mockReturnValue("vault-source"); + (getMiyoFolderPath as jest.Mock).mockReturnValue("/vault"); mockResolveBaseUrl.mockResolvedValue("http://miyo.local"); }); @@ -57,28 +58,28 @@ describe("MiyoSemanticRetriever", () => { { id: "doc-1", score: 0.9, - path: "notes/a.md", + path: "/vault/notes/a.md", chunk_index: 0, chunk_text: "A chunk", }, { id: "doc-1-dup", score: 0.85, - path: "notes/a.md", + path: "/vault/notes/a.md", chunk_index: 0, chunk_text: "A duplicated chunk", }, { id: "doc-2", score: 0.1, - path: "notes/b.md", + path: "/vault/notes/b.md", chunk_index: 0, chunk_text: "Below threshold chunk", }, { id: "doc-3", score: Number.NaN, - path: "notes/c.md", + path: "/vault/notes/c.md", chunk_index: 1, chunk_text: "NaN score chunk should pass", }, @@ -90,7 +91,7 @@ describe("MiyoSemanticRetriever", () => { expect(mockSearch).toHaveBeenCalledWith( "http://miyo.local", - "vault-source", + "/vault", "query with [[notes/a]] mention", 10, undefined @@ -117,7 +118,7 @@ describe("MiyoSemanticRetriever", () => { expect(mockSearch).toHaveBeenCalledWith( "http://miyo.local", - "vault-source", + "/vault", "show notes from this week", 10, [{ field: "mtime", gte: startTime, lte: endTime }] @@ -137,7 +138,7 @@ describe("MiyoSemanticRetriever", () => { expect(mockSearch).toHaveBeenCalledWith( "http://miyo.local", - "vault-source", + "/vault", "list all notes about ai digests", RETURN_ALL_LIMIT, undefined diff --git a/src/search/miyo/MiyoSemanticRetriever.ts b/src/search/miyo/MiyoSemanticRetriever.ts index 2808afd7..2144ecec 100644 --- a/src/search/miyo/MiyoSemanticRetriever.ts +++ b/src/search/miyo/MiyoSemanticRetriever.ts @@ -4,7 +4,7 @@ import { BaseRetriever } from "@langchain/core/retrievers"; import { App } from "obsidian"; import { logInfo, logWarn } from "@/logger"; import { MiyoClient, MiyoSearchFilter, MiyoSearchResult } from "@/miyo/MiyoClient"; -import { getMiyoCustomUrl, getMiyoVault } from "@/miyo/miyoUtils"; +import { getMiyoCustomUrl, getMiyoFolderPath, getVaultRelativeMiyoPath } from "@/miyo/miyoUtils"; import { getSettings } from "@/settings/model"; import { RETURN_ALL_LIMIT } from "@/search/v3/SearchCore"; @@ -91,7 +91,7 @@ export class MiyoSemanticRetriever extends BaseRetriever { } const response = await this.client.search( baseUrl, - getMiyoVault(this.app), + getMiyoFolderPath(this.app), query, limit, filters @@ -140,10 +140,11 @@ export class MiyoSemanticRetriever extends BaseRetriever { * @returns LangChain Document instance. */ private toDocument(result: MiyoSearchResult): Document { + const relativePath = getVaultRelativeMiyoPath(this.app, result.path); const metadata = result.metadata ?? {}; const chunkId = metadata.chunkId || - (result.chunk_index !== undefined ? `${result.path}#${result.chunk_index}` : undefined); + (result.chunk_index !== undefined ? `${relativePath}#${result.chunk_index}` : undefined); const score = typeof result.score === "number" ? result.score.toFixed(2) : "?"; return new Document({ @@ -152,7 +153,7 @@ export class MiyoSemanticRetriever extends BaseRetriever { ...metadata, score: result.score, explanation: `miyo ${score}`, - path: result.path, + path: relativePath, mtime: result.mtime, ctime: result.ctime, title: result.title ?? "", diff --git a/src/search/vectorStoreManager.ts b/src/search/vectorStoreManager.ts index 5b0d7772..abd76532 100644 --- a/src/search/vectorStoreManager.ts +++ b/src/search/vectorStoreManager.ts @@ -130,6 +130,12 @@ export default class VectorStoreManager { return 0; } + if (this.activeBackendKey === "miyo") { + await this.miyoBackend.requestIndexRefresh(Boolean(overwrite)); + notifyIndexChanged(); + return 0; + } + if ( Platform.isMobile && getSettings().disableIndexOnMobile && @@ -281,6 +287,9 @@ export default class VectorStoreManager { public async reindexFile(file: TFile): Promise { await this.waitForInitialization(); + if (this.activeBackendKey === "miyo") { + return; + } await this.indexOps.reindexFile(file); notifyIndexChanged(); } diff --git a/src/settings/model.ts b/src/settings/model.ts index eaabc2b2..eae1e095 100644 --- a/src/settings/model.ts +++ b/src/settings/model.ts @@ -141,7 +141,7 @@ export interface CopilotSettings { selfHostApiKey: string; /** Custom Miyo server URL, e.g. "http://192.168.1.10:8742" (empty = use local service discovery) */ miyoServerUrl: string; - /** Remote vault path sent to Miyo as the vault's source identifier (overrides auto-detected vault path). Only relevant when a remote Miyo server URL is configured. */ + /** @deprecated Miyo now uses the current vault path as `folder_path`; preserved only for backwards-compatible settings migration. */ miyoRemoteVaultPath: string; /** Which provider to use for self-host web search */ selfHostSearchProvider: "firecrawl" | "perplexity"; diff --git a/src/settings/v2/components/CopilotPlusSettings.tsx b/src/settings/v2/components/CopilotPlusSettings.tsx index 6db8a8f3..dc0dda38 100644 --- a/src/settings/v2/components/CopilotPlusSettings.tsx +++ b/src/settings/v2/components/CopilotPlusSettings.tsx @@ -4,7 +4,7 @@ import { HelpTooltip } from "@/components/ui/help-tooltip"; import { SettingItem } from "@/components/ui/setting-item"; import { DEFAULT_SETTINGS } from "@/constants"; import { MiyoClient } from "@/miyo/MiyoClient"; -import { getMiyoCustomUrl, resolveMiyoVault } from "@/miyo/miyoUtils"; +import { getMiyoCustomUrl, getMiyoFolderPath } from "@/miyo/miyoUtils"; import { useIsSelfHostEligible, validateSelfHostMode } from "@/plusUtils"; import { updateSetting, useSettingsValue } from "@/settings/model"; import { Notice } from "obsidian"; @@ -86,8 +86,8 @@ export const CopilotPlusSettings: React.FC = () => { new ConfirmModal( app, confirmChange, - `Enabling Miyo Search will refresh your vault index to store data in Miyo. Embedding Batch Size will be reset to the default (${DEFAULT_SETTINGS.embeddingBatchSize}) for local stability. If your hardware is strong, you can increase this later in QA settings. Continue?`, - "Refresh Index" + `Enabling Miyo Search will use your current vault path as the Miyo folder path and request a scan from Miyo. Make sure this folder is already registered in Miyo. Embedding Batch Size will be reset to the default (${DEFAULT_SETTINGS.embeddingBatchSize}) for local stability. Continue?`, + "Request Miyo Scan" ).open(); }; @@ -227,24 +227,10 @@ export const CopilotPlusSettings: React.FC = () => { onChange={(value) => updateSetting("miyoServerUrl", value)} /> - - Leave blank when accessing Miyo locally. Set this only when using a remote - Miyo server to override the vault identifier. Must match exactly across all - devices sharing the same remote Miyo instance. - - } - value={settings.miyoRemoteVaultPath || ""} - onChange={(value) => updateSetting("miyoRemoteVaultPath", value)} - /> - { {settings.enableMiyo && (
- Vault path:{" "} + Folder path sent to Miyo:{" "} - {resolveMiyoVault( - app, - (settings.miyoServerUrl || "").trim() - ? settings.miyoRemoteVaultPath || "" - : "" - )} + {getMiyoFolderPath(app)}
)} diff --git a/src/tools/SearchTools.ts b/src/tools/SearchTools.ts index cff90cb7..6566d2db 100644 --- a/src/tools/SearchTools.ts +++ b/src/tools/SearchTools.ts @@ -3,6 +3,7 @@ import { TEXT_WEIGHT } from "@/constants"; import { BrevilabsClient } from "@/LLMProviders/brevilabsClient"; import { hasSelfHostSearchKey, selfHostWebSearch } from "@/LLMProviders/selfHostServices"; import { logInfo } from "@/logger"; +import { shouldUseMiyo } from "@/miyo/miyoUtils"; import { isSelfHostModeValid } from "@/plusUtils"; import { RetrieverFactory } from "@/search/RetrieverFactory"; import { getSettings } from "@/settings/model"; @@ -509,11 +510,17 @@ const indexTool = createLangChainTool({ try { const VectorStoreManager = (await import("@/search/vectorStoreManager")).default; const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(); - const indexResultPrompt = `Semantic search index refreshed with ${count} documents.\n`; + const usingMiyo = shouldUseMiyo(settings); + const indexResultPrompt = usingMiyo + ? "Requested a Miyo folder scan for this vault.\n" + : `Semantic search index refreshed with ${count} documents.\n`; return { success: true, - message: - indexResultPrompt + `Semantic search index has been refreshed with ${count} documents.`, + message: usingMiyo + ? indexResultPrompt + + "Miyo will handle chunking and indexing for the registered folder." + : indexResultPrompt + + `Semantic search index has been refreshed with ${count} documents.`, documentCount: count, }; } catch (error: any) {