mirror of
https://github.com/logancyang/obsidian-copilot.git
synced 2026-07-22 07:50:24 +00:00
Refactor Miyo integration for folder API (#2331)
* refactor Miyo integration for folder API * chore: remove Miyo related-search log * docs: add Miyo Node Service API reference Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Wenzheng Jiang
GitHub
parent
c0d79efb7c
commit
47e61259d2
16 changed files with 969 additions and 401 deletions
471
docs/miyo-api.md
Normal file
471
docs/miyo-api.md
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
# Miyo Node Service API
|
||||
|
||||
Base URL: `http://127.0.0.1:8742`
|
||||
|
||||
All request and response bodies are JSON. Errors always return `{ "detail": "<message>" }`.
|
||||
|
||||
---
|
||||
|
||||
## Health
|
||||
|
||||
### `GET /v0/health`
|
||||
|
||||
Returns service and sidecar status.
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ok | degraded",
|
||||
"service": "running",
|
||||
"qdrant": "connected | ...",
|
||||
"llama_server": "running | ...",
|
||||
"model_download_progress": 0.75,
|
||||
"embedding_model": "nomic-embed-text-v1.5",
|
||||
"batch_size_preset": "default",
|
||||
"gpu_variant": "metal | null",
|
||||
"indexed_files": 1234
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Search
|
||||
|
||||
### `POST /v0/search`
|
||||
|
||||
Hybrid semantic + keyword search (dense + BM25, fused via RRF).
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "string (required)",
|
||||
"folder_path": "string | null — restrict to this folder",
|
||||
"path": "string | null — substring filter on file path (case-insensitive)",
|
||||
"limit": 10,
|
||||
"filters": [
|
||||
/* MetadataFilter[], see below */
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{
|
||||
"results": [
|
||||
/* SearchResult[] */
|
||||
],
|
||||
"query": "string",
|
||||
"count": 5,
|
||||
"execution_time_ms": 42.0
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:** 400 (missing query), 503 (llama-server or Qdrant unavailable)
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/search/related`
|
||||
|
||||
Find files related to a given file using vector similarity.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{
|
||||
"file_path": "string (required) — absolute path",
|
||||
"folder_path": "string | null",
|
||||
"limit": 10,
|
||||
"filters": [
|
||||
/* MetadataFilter[] */
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{
|
||||
"results": [{ "path": "string", "score": 0.95 }],
|
||||
"file_path": "string",
|
||||
"count": 5,
|
||||
"execution_time_ms": 12.0
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:** 400, 404 (no indexed chunks for the file), 503
|
||||
|
||||
---
|
||||
|
||||
## Folders
|
||||
|
||||
### `GET /v0/folder`
|
||||
|
||||
- With `?path=<folder_path>`: returns a single `FolderEntry` (404 if not registered)
|
||||
- Without `path`: returns `{ "folders": [ FolderEntry[] ] }`
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/folder`
|
||||
|
||||
Register a folder for indexing. Starts watching and scanning immediately.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "string (required) — absolute path",
|
||||
"include_patterns": ["**/*.md"],
|
||||
"exclude_patterns": ["**/node_modules/**"],
|
||||
"recursive": true
|
||||
}
|
||||
```
|
||||
|
||||
**Response 201** — `FolderEntry`
|
||||
|
||||
**Errors:** 400 (invalid), 409 (already registered)
|
||||
|
||||
---
|
||||
|
||||
### `PATCH /v0/folder`
|
||||
|
||||
Update folder configuration. Only provided fields are changed.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "string (required)",
|
||||
"include_patterns": ["**/*.md"],
|
||||
"exclude_patterns": ["**/node_modules/**"],
|
||||
"recursive": false
|
||||
}
|
||||
```
|
||||
|
||||
**Response 200** — updated `FolderEntry`
|
||||
|
||||
**Errors:** 400, 404
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v0/folder`
|
||||
|
||||
Unregister a folder and remove all its indexed data.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{ "path": "string (required)" }
|
||||
```
|
||||
|
||||
**Response 200** — deletion summary object
|
||||
|
||||
**Errors:** 400, 404
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/folder/pause`
|
||||
|
||||
Stop file watching for a folder without removing it.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{ "path": "string (required)" }
|
||||
```
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{ "status": "paused", "path": "string" }
|
||||
```
|
||||
|
||||
**Errors:** 400, 404
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/folder/resume`
|
||||
|
||||
Resume file watching and trigger a rescan.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{ "path": "string (required)" }
|
||||
```
|
||||
|
||||
**Response 202**
|
||||
|
||||
```json
|
||||
{ "status": "scanning", "path": "string" }
|
||||
```
|
||||
|
||||
**Errors:** 400, 404
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/scan`
|
||||
|
||||
Manually trigger a rescan of a registered folder.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "string (required)",
|
||||
"force": false
|
||||
}
|
||||
```
|
||||
|
||||
`force: true` re-indexes all files even if unchanged.
|
||||
|
||||
**Response 202**
|
||||
|
||||
```json
|
||||
{ "status": "started", "path": "string" }
|
||||
```
|
||||
|
||||
**Errors:** 400, 404
|
||||
|
||||
---
|
||||
|
||||
## Files & Documents
|
||||
|
||||
### `GET /v0/folder/files`
|
||||
|
||||
List indexed files with optional filtering and pagination.
|
||||
|
||||
**Query parameters**
|
||||
|
||||
| Param | Type | Description |
|
||||
| -------------- | --------------------------------- | ------------------------------- |
|
||||
| `folder_path` | string | Filter by folder |
|
||||
| `title` | string | Substring match on title |
|
||||
| `file_path` | string | Exact file path match |
|
||||
| `mtime_after` | number | Unix timestamp lower bound |
|
||||
| `mtime_before` | number | Unix timestamp upper bound |
|
||||
| `offset` | integer (default 0) | Pagination offset |
|
||||
| `limit` | integer | Max results (omit for no limit) |
|
||||
| `order_by` | `mtime` \| `updated_at` (default) | Sort order |
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{
|
||||
"files": [
|
||||
/* FileEntry[] */
|
||||
],
|
||||
"total": 99
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v0/folder/documents`
|
||||
|
||||
Fetch all indexed chunks for a specific file, sorted by chunk index.
|
||||
|
||||
**Query parameters**
|
||||
|
||||
| Param | Required | Description |
|
||||
| ------------- | -------- | -------------------------- |
|
||||
| `path` | yes | Absolute file path |
|
||||
| `folder_path` | no | Scope to a specific folder |
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{
|
||||
"documents": [
|
||||
/* DocumentChunk[] */
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:** 400, 503
|
||||
|
||||
---
|
||||
|
||||
## Utilities
|
||||
|
||||
### `POST /v0/parse-doc`
|
||||
|
||||
Parse a file and return its extracted text content.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{ "path": "string (required) — absolute file path" }
|
||||
```
|
||||
|
||||
**Response 200** — parsed content object (shape varies by file type)
|
||||
|
||||
**Errors:**
|
||||
|
||||
| Code | Meaning |
|
||||
| ---- | --------------------- |
|
||||
| 400 | Invalid input |
|
||||
| 403 | File not readable |
|
||||
| 404 | File not found |
|
||||
| 415 | Unsupported file type |
|
||||
| 422 | Parse failed |
|
||||
| 500 | Internal error |
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/rebuild-metadata`
|
||||
|
||||
Rebuild the manifest by re-syncing metadata from Qdrant. Use when manifest is out of sync.
|
||||
|
||||
**Response 200** — `{ "elapsed_ms": 123, ...stats }`
|
||||
|
||||
**Errors:** 409 (rebuild already in progress), 503
|
||||
|
||||
---
|
||||
|
||||
### `POST /v0/llama-server/restart`
|
||||
|
||||
Restart the llama-server sidecar, optionally changing the batch size preset.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{ "batch_size_preset": "default" }
|
||||
```
|
||||
|
||||
**Response 200**
|
||||
|
||||
```json
|
||||
{
|
||||
"restarted": true,
|
||||
"batch_size_preset": "default",
|
||||
"status": "running"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:** 400
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/embeddings`
|
||||
|
||||
Generate embeddings. OpenAI-compatible interface, proxied to llama-server.
|
||||
|
||||
**Request body**
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "nomic-embed-text-v1.5",
|
||||
"input": "string or string[]"
|
||||
}
|
||||
```
|
||||
|
||||
`model` is optional but must match the configured embedding model if provided.
|
||||
|
||||
**Response 200** — standard OpenAI embeddings response
|
||||
|
||||
**Errors:** 400, 503
|
||||
|
||||
---
|
||||
|
||||
## Schemas
|
||||
|
||||
### MetadataFilter
|
||||
|
||||
Range filter on a metadata field.
|
||||
|
||||
```json
|
||||
{
|
||||
"field": "mtime",
|
||||
"gt": 1700000000,
|
||||
"gte": 1700000000,
|
||||
"lt": 1800000000,
|
||||
"lte": 1800000000
|
||||
}
|
||||
```
|
||||
|
||||
- `field` can be `mtime`, `ctime`, or any metadata key
|
||||
- Bare field names (not `mtime`/`ctime` and not prefixed with `metadata.`) are automatically prefixed with `metadata.`
|
||||
- At least one of `gt`, `gte`, `lt`, `lte` must be present
|
||||
|
||||
---
|
||||
|
||||
### SearchResult
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "string",
|
||||
"score": 0.95,
|
||||
"title": "string | null",
|
||||
"mtime": 1700000000,
|
||||
"ctime": 1700000000,
|
||||
"file_name": "string | null",
|
||||
"chunk_index": 0,
|
||||
"total_chunks": 5,
|
||||
"chunk_text": "string | null",
|
||||
"metadata": {},
|
||||
"embedding_model": "string | null",
|
||||
"tags": ["string"],
|
||||
"extension": ".md",
|
||||
"created_at": "string | null",
|
||||
"nchars": 1024,
|
||||
"folder_path": "string | null"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### FileEntry
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "string",
|
||||
"title": "string | null",
|
||||
"mtime": 1700000000,
|
||||
"updated_at": "ISO8601 string",
|
||||
"folder_path": "string | null",
|
||||
"total_chunks": 5
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### DocumentChunk
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "string",
|
||||
"path": "string | null",
|
||||
"title": "string | null",
|
||||
"chunk_index": 0,
|
||||
"chunk_text": "string | null",
|
||||
"metadata": {},
|
||||
"embedding_model": "string | null",
|
||||
"ctime": 1700000000,
|
||||
"mtime": 1700000000,
|
||||
"tags": ["string"],
|
||||
"extension": ".md",
|
||||
"created_at": "ISO8601 string | null",
|
||||
"nchars": 1024,
|
||||
"folder_path": "string | null"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### FolderEntry
|
||||
|
||||
Shape varies — includes at minimum:
|
||||
|
||||
```json
|
||||
{
|
||||
"path": "string",
|
||||
"include_patterns": ["**/*.md"],
|
||||
"exclude_patterns": [],
|
||||
"recursive": true
|
||||
}
|
||||
```
|
||||
|
||||
Plus live stats fields populated by the folder manager.
|
||||
|
|
@ -23,6 +23,7 @@ import { YoutubeTranscriptModal } from "@/components/modals/YoutubeTranscriptMod
|
|||
import { checkIsPlusUser } from "@/plusUtils";
|
||||
// Debug modals removed with search v3
|
||||
import CopilotPlugin from "@/main";
|
||||
import { shouldUseMiyo } from "@/miyo/miyoUtils";
|
||||
import { getAllQAMarkdownContent } from "@/search/searchUtils";
|
||||
import { CopilotSettings } from "@/settings/model";
|
||||
import { NoteSelectedTextContext, WebSelectedTextContext } from "@/types/message";
|
||||
|
|
@ -179,10 +180,16 @@ export function registerCommands(
|
|||
|
||||
addCommand(plugin, COMMAND_IDS.CLEAR_LOCAL_COPILOT_INDEX, async () => {
|
||||
const { getSettings } = await import("@/settings/model");
|
||||
const isMiyoEnabled = getSettings().enableMiyo;
|
||||
const clearMessage = isMiyoEnabled
|
||||
? "This will permanently delete all indexes of this vault from Miyo. This action cannot be undone.\n\nAre you sure you want to proceed?"
|
||||
: "This will permanently delete all document indexes in Copilot. This action cannot be undone.\n\nAre you sure you want to proceed?";
|
||||
const settings = getSettings();
|
||||
const isMiyoEnabled = shouldUseMiyo(settings);
|
||||
if (isMiyoEnabled) {
|
||||
new Notice(
|
||||
"Miyo folders are managed in Miyo. Remove the folder there if you want to clear it."
|
||||
);
|
||||
return;
|
||||
}
|
||||
const clearMessage =
|
||||
"This will permanently delete all document indexes in Copilot. This action cannot be undone.\n\nAre you sure you want to proceed?";
|
||||
const confirmed = await new Promise<boolean>((resolve) => {
|
||||
new ConfirmModal(
|
||||
plugin.app,
|
||||
|
|
@ -207,6 +214,13 @@ export function registerCommands(
|
|||
|
||||
addCommand(plugin, COMMAND_IDS.GARBAGE_COLLECT_COPILOT_INDEX, async () => {
|
||||
try {
|
||||
const { getSettings } = await import("@/settings/model");
|
||||
if (shouldUseMiyo(getSettings())) {
|
||||
new Notice(
|
||||
"Miyo manages file cleanup automatically. Run Index (refresh) vault to trigger a scan if needed."
|
||||
);
|
||||
return;
|
||||
}
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
const removedCount = await VectorStoreManager.getInstance().garbageCollectVectorStore();
|
||||
new Notice(`Garbage collection completed. Removed ${removedCount} stale documents.`);
|
||||
|
|
@ -229,7 +243,11 @@ export function registerCommands(
|
|||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(false, {
|
||||
userInitiated: true,
|
||||
});
|
||||
new Notice(`Semantic search index refreshed with ${count} documents.`);
|
||||
if (shouldUseMiyo(settings)) {
|
||||
new Notice("Miyo folder index refresh started. Open the Miyo app to check details.");
|
||||
} else {
|
||||
new Notice(`Semantic search index refreshed with ${count} documents.`);
|
||||
}
|
||||
} else {
|
||||
// V3 search builds indexes on demand
|
||||
new Notice("Lexical search builds indexes on demand. No manual indexing required.");
|
||||
|
|
@ -263,7 +281,11 @@ export function registerCommands(
|
|||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(true, {
|
||||
userInitiated: true,
|
||||
});
|
||||
new Notice(`Semantic search index rebuilt with ${count} documents.`);
|
||||
if (shouldUseMiyo(settings)) {
|
||||
new Notice("Miyo folder index refresh started. Open the Miyo app to check details.");
|
||||
} else {
|
||||
new Notice(`Semantic search index rebuilt with ${count} documents.`);
|
||||
}
|
||||
} else {
|
||||
// V3 search builds indexes on demand
|
||||
new Notice("Lexical search builds indexes on demand. No manual indexing required.");
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { SettingSwitch } from "@/components/ui/setting-switch";
|
|||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { PLUS_UTM_MEDIUMS } from "@/constants";
|
||||
import { logError } from "@/logger";
|
||||
import { shouldUseMiyo } from "@/miyo/miyoUtils";
|
||||
import { navigateToPlusPage, useIsPlusUser } from "@/plusUtils";
|
||||
import { updateSetting, useSettingsValue } from "@/settings/model";
|
||||
import { Docs4LLMParser } from "@/tools/FileParserManager";
|
||||
|
|
@ -44,9 +45,14 @@ export async function refreshVaultIndex() {
|
|||
if (settings.enableSemanticSearchV3) {
|
||||
// Use VectorStoreManager for semantic search indexing
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
await VectorStoreManager.getInstance().indexVaultToVectorStore(false, {
|
||||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(false, {
|
||||
userInitiated: true,
|
||||
});
|
||||
if (shouldUseMiyo(settings)) {
|
||||
new Notice("Miyo folder index refresh started. Open the Miyo app to check details.");
|
||||
} else {
|
||||
new Notice(`Semantic search index refreshed with ${count} documents.`);
|
||||
}
|
||||
} else {
|
||||
// V3 search builds indexes on demand
|
||||
new Notice("Lexical search builds indexes on demand. No manual indexing required.");
|
||||
|
|
@ -65,7 +71,14 @@ export async function forceReindexVault() {
|
|||
if (settings.enableSemanticSearchV3) {
|
||||
// Use VectorStoreManager for semantic search indexing
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
await VectorStoreManager.getInstance().indexVaultToVectorStore(true, { userInitiated: true });
|
||||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore(true, {
|
||||
userInitiated: true,
|
||||
});
|
||||
if (shouldUseMiyo(settings)) {
|
||||
new Notice("Miyo folder index refresh started. Open the Miyo app to check details.");
|
||||
} else {
|
||||
new Notice(`Semantic search index rebuilt with ${count} documents.`);
|
||||
}
|
||||
} else {
|
||||
// V3 search builds indexes on demand
|
||||
new Notice("Lexical search builds indexes on demand. No manual indexing required.");
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { getDecryptedKey } from "@/encryptionService";
|
||||
import { logInfo } from "@/logger";
|
||||
import { MiyoClient } from "@/miyo/MiyoClient";
|
||||
import { MiyoServiceDiscovery } from "@/miyo/MiyoServiceDiscovery";
|
||||
|
|
@ -12,6 +13,10 @@ jest.mock("@/settings/model", () => ({
|
|||
getSettings: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock("@/encryptionService", () => ({
|
||||
getDecryptedKey: jest.fn(async (value: string) => value),
|
||||
}));
|
||||
|
||||
const mockResolveBaseUrl = jest.fn();
|
||||
|
||||
jest.mock("@/miyo/MiyoServiceDiscovery", () => ({
|
||||
|
|
@ -28,11 +33,12 @@ jest.mock("@/logger", () => ({
|
|||
logError: jest.fn(),
|
||||
}));
|
||||
|
||||
describe("MiyoClient.parseDoc", () => {
|
||||
describe("MiyoClient", () => {
|
||||
const mockedRequestUrl = requestUrl as jest.MockedFunction<typeof requestUrl>;
|
||||
const mockedGetSettings = getSettings as jest.MockedFunction<typeof getSettings>;
|
||||
const mockedGetInstance = MiyoServiceDiscovery.getInstance as unknown as jest.Mock;
|
||||
const mockedLogInfo = logInfo as jest.MockedFunction<typeof logInfo>;
|
||||
const mockedGetDecryptedKey = getDecryptedKey as jest.MockedFunction<typeof getDecryptedKey>;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
|
@ -40,6 +46,7 @@ describe("MiyoClient.parseDoc", () => {
|
|||
plusLicenseKey: "plus-test-license",
|
||||
debug: false,
|
||||
} as any);
|
||||
mockedGetDecryptedKey.mockResolvedValue("plus-test-license");
|
||||
mockResolveBaseUrl.mockResolvedValue("http://127.0.0.1:8742");
|
||||
mockedGetInstance.mockReturnValue({
|
||||
resolveBaseUrl: mockResolveBaseUrl,
|
||||
|
|
@ -90,17 +97,86 @@ describe("MiyoClient.parseDoc", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("throws when /v0/parse-doc returns an error status", async () => {
|
||||
it("sends folder_path in /v0/search requests", async () => {
|
||||
mockedRequestUrl.mockResolvedValue({
|
||||
status: 500,
|
||||
text: "internal server error",
|
||||
json: { detail: "fail" },
|
||||
status: 200,
|
||||
json: { results: [] },
|
||||
text: "",
|
||||
} as any);
|
||||
|
||||
const client = new MiyoClient();
|
||||
await client.search("http://127.0.0.1:8742", "/vault", "project notes", 10, [
|
||||
{ field: "mtime", gte: 1, lte: 2 },
|
||||
]);
|
||||
|
||||
expect(mockedRequestUrl).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8742/v0/search",
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
query: "project notes",
|
||||
folder_path: "/vault",
|
||||
limit: 10,
|
||||
filters: [{ field: "mtime", gte: 1, lte: 2 }],
|
||||
}),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("requests folder scans through /v0/scan", async () => {
|
||||
mockedRequestUrl.mockResolvedValue({
|
||||
status: 202,
|
||||
json: { status: "started", path: "/vault" },
|
||||
text: "",
|
||||
} as any);
|
||||
|
||||
const client = new MiyoClient();
|
||||
const result = await client.scanFolder("http://127.0.0.1:8742", "/vault", true);
|
||||
|
||||
expect(result).toEqual({ status: "started", path: "/vault" });
|
||||
expect(mockedRequestUrl).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8742/v0/scan",
|
||||
method: "POST",
|
||||
body: JSON.stringify({ path: "/vault", force: true }),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("lists indexed files from /v0/folder/files with folder_path query params", async () => {
|
||||
mockedRequestUrl.mockResolvedValue({
|
||||
status: 200,
|
||||
json: { files: [], total: 0 },
|
||||
text: "",
|
||||
} as any);
|
||||
|
||||
const client = new MiyoClient();
|
||||
await client.listFolderFiles("http://127.0.0.1:8742", {
|
||||
folderPath: "/vault",
|
||||
offset: 10,
|
||||
limit: 25,
|
||||
orderBy: "mtime",
|
||||
});
|
||||
|
||||
expect(mockedRequestUrl).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8742/v0/folder/files?folder_path=%2Fvault&offset=10&limit=25&order_by=mtime",
|
||||
method: "GET",
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("throws detailed errors when a request fails", async () => {
|
||||
mockedRequestUrl.mockResolvedValue({
|
||||
status: 404,
|
||||
text: "not found",
|
||||
json: { detail: "folder not registered" },
|
||||
} as any);
|
||||
|
||||
const client = new MiyoClient();
|
||||
|
||||
await expect(client.parseDoc("http://127.0.0.1:8742", "/tmp/sample.pdf")).rejects.toThrow(
|
||||
"Miyo request failed with status 500"
|
||||
await expect(client.getFolder("http://127.0.0.1:8742", "/vault")).rejects.toThrow(
|
||||
"Miyo request failed with status 404: folder not registered"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,44 +1,47 @@
|
|||
import { getDecryptedKey } from "@/encryptionService";
|
||||
import { logError, logInfo, logWarn } from "@/logger";
|
||||
import { MiyoServiceDiscovery } from "@/miyo/MiyoServiceDiscovery";
|
||||
import { getSettings } from "@/settings/model";
|
||||
import { err2String } from "@/utils";
|
||||
import { requestUrl } from "obsidian";
|
||||
import { MiyoServiceDiscovery } from "@/miyo/MiyoServiceDiscovery";
|
||||
|
||||
/**
|
||||
* Request payload for upserting documents into Miyo.
|
||||
* Indexed file entry returned by Miyo.
|
||||
*/
|
||||
export interface MiyoUpsertDocument {
|
||||
id: string;
|
||||
export interface MiyoIndexedFileEntry {
|
||||
path: string;
|
||||
title: string;
|
||||
content: string;
|
||||
created_at: number;
|
||||
ctime: number;
|
||||
title?: string | null;
|
||||
mtime: number;
|
||||
tags: string[];
|
||||
extension: string;
|
||||
nchars: number;
|
||||
metadata: Record<string, unknown>;
|
||||
updated_at?: string;
|
||||
folder_path?: string | null;
|
||||
total_chunks?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Response for index file listing.
|
||||
* Response for indexed file listing.
|
||||
*/
|
||||
export interface MiyoIndexedFilesResponse {
|
||||
files: Array<{ path: string; mtime: number }>;
|
||||
files: MiyoIndexedFileEntry[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Response for index statistics.
|
||||
* Folder entry returned by Miyo.
|
||||
*/
|
||||
export interface MiyoIndexStatsResponse {
|
||||
total_chunks: number;
|
||||
total_files: number;
|
||||
latest_mtime: number;
|
||||
embedding_model?: string;
|
||||
embedding_dim?: number;
|
||||
export interface MiyoFolderEntry {
|
||||
path: string;
|
||||
include_patterns?: string[];
|
||||
exclude_patterns?: string[];
|
||||
recursive?: boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Response for scan requests.
|
||||
*/
|
||||
export interface MiyoScanResponse {
|
||||
status?: string;
|
||||
path?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -48,17 +51,18 @@ export interface MiyoDocumentsResponse {
|
|||
documents: Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
title?: string;
|
||||
title?: string | null;
|
||||
chunk_index?: number;
|
||||
chunk_text?: string;
|
||||
chunk_text?: string | null;
|
||||
metadata?: Record<string, unknown>;
|
||||
embedding_model?: string;
|
||||
embedding_model?: string | null;
|
||||
ctime?: number;
|
||||
mtime?: number;
|
||||
tags?: string[];
|
||||
extension?: string;
|
||||
created_at?: number;
|
||||
created_at?: string | number | null;
|
||||
nchars?: number;
|
||||
folder_path?: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
|
|
@ -69,17 +73,18 @@ export interface MiyoSearchResult {
|
|||
id: string;
|
||||
score: number;
|
||||
path: string;
|
||||
title?: string;
|
||||
title?: string | null;
|
||||
chunk_index?: number;
|
||||
chunk_text: string;
|
||||
chunk_text?: string | null;
|
||||
metadata?: Record<string, unknown>;
|
||||
embedding_model?: string;
|
||||
embedding_model?: string | null;
|
||||
ctime?: number;
|
||||
mtime?: number;
|
||||
tags?: string[];
|
||||
extension?: string;
|
||||
created_at?: number;
|
||||
created_at?: string | number | null;
|
||||
nchars?: number;
|
||||
folder_path?: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -179,113 +184,103 @@ export class MiyoClient {
|
|||
}
|
||||
|
||||
/**
|
||||
* Upsert a batch of documents.
|
||||
* Fetch the Miyo folder entry for a registered folder path.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
* @param documents - Documents to upsert.
|
||||
* @returns Number of documents upserted.
|
||||
* @param folderPath - Absolute folder path registered in Miyo.
|
||||
* @returns Folder entry.
|
||||
*/
|
||||
public async upsertDocuments(
|
||||
public async getFolder(baseUrl: string, folderPath: string): Promise<MiyoFolderEntry> {
|
||||
return this.requestJson<MiyoFolderEntry>(baseUrl, "/v0/folder", {
|
||||
method: "GET",
|
||||
query: { path: folderPath },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a Miyo folder scan.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param folderPath - Absolute folder path registered in Miyo.
|
||||
* @param force - Whether to force a full re-scan.
|
||||
* @returns Scan response.
|
||||
*/
|
||||
public async scanFolder(
|
||||
baseUrl: string,
|
||||
vault: string,
|
||||
documents: MiyoUpsertDocument[]
|
||||
): Promise<number> {
|
||||
const payload = { vault, documents };
|
||||
const response = await this.requestJson<{ upserted: number }>(baseUrl, "/v0/index/upsert", {
|
||||
folderPath: string,
|
||||
force = false
|
||||
): Promise<MiyoScanResponse> {
|
||||
return this.requestJson<MiyoScanResponse>(baseUrl, "/v0/scan", {
|
||||
method: "POST",
|
||||
body: payload,
|
||||
body: {
|
||||
path: folderPath,
|
||||
force,
|
||||
},
|
||||
});
|
||||
return response?.upserted ?? 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete documents by file path.
|
||||
* List indexed files for a folder with pagination and filters.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
* @param path - File path to delete.
|
||||
* @returns Count of documents deleted.
|
||||
*/
|
||||
public async deleteByPath(baseUrl: string, vault: string, path: string): Promise<number> {
|
||||
const payload = { vault, path };
|
||||
const response = await this.requestJson<{ deleted?: number }>(baseUrl, "/v0/index/by_path", {
|
||||
method: "DELETE",
|
||||
body: payload,
|
||||
});
|
||||
return response?.deleted ?? 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all indexed documents for a source.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
*/
|
||||
public async clearIndex(baseUrl: string, vault: string): Promise<void> {
|
||||
const payload = { vault };
|
||||
await this.requestJson(baseUrl, "/v0/index/clear", { method: "POST", body: payload });
|
||||
}
|
||||
|
||||
/**
|
||||
* List indexed files with pagination.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
* @param offset - Offset for pagination.
|
||||
* @param limit - Page size.
|
||||
* @param options - Folder file list options.
|
||||
* @returns Indexed files response.
|
||||
*/
|
||||
public async listFiles(
|
||||
public async listFolderFiles(
|
||||
baseUrl: string,
|
||||
vault: string,
|
||||
offset: number,
|
||||
limit: number
|
||||
options: {
|
||||
folderPath: string;
|
||||
title?: string;
|
||||
filePath?: string;
|
||||
mtimeAfter?: number;
|
||||
mtimeBefore?: number;
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
orderBy?: "mtime" | "updated_at";
|
||||
}
|
||||
): Promise<MiyoIndexedFilesResponse> {
|
||||
return this.requestJson<MiyoIndexedFilesResponse>(baseUrl, "/v0/index/files", {
|
||||
return this.requestJson<MiyoIndexedFilesResponse>(baseUrl, "/v0/folder/files", {
|
||||
method: "GET",
|
||||
query: { vault, offset, limit },
|
||||
query: {
|
||||
folder_path: options.folderPath,
|
||||
title: options.title,
|
||||
file_path: options.filePath,
|
||||
mtime_after: options.mtimeAfter,
|
||||
mtime_before: options.mtimeBefore,
|
||||
offset: options.offset,
|
||||
limit: options.limit,
|
||||
order_by: options.orderBy,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch index statistics for a source.
|
||||
* Fetch all indexed chunks for a file path.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
* @returns Index stats response.
|
||||
*/
|
||||
public async getStats(baseUrl: string, vault: string): Promise<MiyoIndexStatsResponse> {
|
||||
return this.requestJson<MiyoIndexStatsResponse>(baseUrl, "/v0/index/stats", {
|
||||
method: "GET",
|
||||
query: { vault },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all documents for a given path.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
* @param path - File path to look up.
|
||||
* @param folderPath - Folder path to scope the document lookup.
|
||||
* @param path - Absolute file path to look up.
|
||||
* @returns Documents response.
|
||||
*/
|
||||
public async getDocumentsByPath(
|
||||
baseUrl: string,
|
||||
vault: string,
|
||||
folderPath: string,
|
||||
path: string
|
||||
): Promise<MiyoDocumentsResponse> {
|
||||
return this.requestJson<MiyoDocumentsResponse>(baseUrl, "/v0/index/documents", {
|
||||
return this.requestJson<MiyoDocumentsResponse>(baseUrl, "/v0/folder/documents", {
|
||||
method: "GET",
|
||||
query: { vault, path },
|
||||
query: {
|
||||
path,
|
||||
folder_path: folderPath,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a hybrid search query.
|
||||
* Execute a hybrid search query scoped to a folder path.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param vault - Vault identifier sent to Miyo.
|
||||
* @param folderPath - Folder path sent to Miyo.
|
||||
* @param query - User query.
|
||||
* @param limit - Maximum number of results.
|
||||
* @param filters - Optional search filters.
|
||||
|
|
@ -293,14 +288,14 @@ export class MiyoClient {
|
|||
*/
|
||||
public async search(
|
||||
baseUrl: string,
|
||||
vault: string,
|
||||
folderPath: string,
|
||||
query: string,
|
||||
limit: number,
|
||||
filters?: MiyoSearchFilter[]
|
||||
): Promise<MiyoSearchResponse> {
|
||||
const payload = {
|
||||
query,
|
||||
vault,
|
||||
folder_path: folderPath,
|
||||
limit,
|
||||
...(filters && filters.length > 0 ? { filters } : {}),
|
||||
};
|
||||
|
|
@ -317,28 +312,25 @@ export class MiyoClient {
|
|||
* Execute related-notes search for a source note path.
|
||||
*
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param filePath - Source note path to find related notes for.
|
||||
* @param options - Optional source id, result limit, and filters.
|
||||
* @param filePath - Absolute source note path to find related notes for.
|
||||
* @param options - Optional folder path, result limit, and filters.
|
||||
* @returns Search response in the same shape as /v0/search.
|
||||
*/
|
||||
public async searchRelated(
|
||||
baseUrl: string,
|
||||
filePath: string,
|
||||
options?: {
|
||||
vault?: string;
|
||||
folderPath?: string;
|
||||
limit?: number;
|
||||
filters?: MiyoSearchFilter[];
|
||||
}
|
||||
): Promise<MiyoRelatedSearchResponse> {
|
||||
const payload = {
|
||||
file_path: filePath,
|
||||
...(options?.vault ? { vault: options.vault } : {}),
|
||||
...(options?.folderPath ? { folder_path: options.folderPath } : {}),
|
||||
...(typeof options?.limit === "number" ? { limit: options.limit } : {}),
|
||||
...(options?.filters && options.filters.length > 0 ? { filters: options.filters } : {}),
|
||||
};
|
||||
if (getSettings().debug) {
|
||||
logInfo("Miyo related search request:", { baseUrl, payload });
|
||||
}
|
||||
return this.requestJson<MiyoRelatedSearchResponse>(baseUrl, "/v0/search/related", {
|
||||
method: "POST",
|
||||
body: payload,
|
||||
|
|
@ -415,9 +407,6 @@ export class MiyoClient {
|
|||
...(getSettings().debug && options.method === "POST" ? { postBody: options.body } : {}),
|
||||
});
|
||||
|
||||
// TODO: Add a configurable timeout for large file parsing (e.g. big PDFs).
|
||||
// Obsidian's requestUrl does not expose a timeout option, so consider using
|
||||
// AbortController or a wrapper with Promise.race to prevent indefinite hangs.
|
||||
const response = await requestUrl({
|
||||
url: url.toString(),
|
||||
method: options.method,
|
||||
|
|
@ -428,9 +417,17 @@ export class MiyoClient {
|
|||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
const errorText = response.text ? response.text : "";
|
||||
const errorPayload = this.parseResponseJson<{ detail?: string }>(
|
||||
response.json,
|
||||
response.text
|
||||
);
|
||||
const errorText = errorPayload?.detail || response.text || "";
|
||||
logWarn(`Miyo request failed (${response.status}): ${errorText}`);
|
||||
throw new Error(`Miyo request failed with status ${response.status}`);
|
||||
throw new Error(
|
||||
errorText
|
||||
? `Miyo request failed with status ${response.status}: ${errorText}`
|
||||
: `Miyo request failed with status ${response.status}`
|
||||
);
|
||||
}
|
||||
|
||||
const parsed = this.parseResponseJson<T>(response.json, response.text);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { isSelfHostAccessValid } from "@/plusUtils";
|
||||
import { CopilotSettings, getSettings } from "@/settings/model";
|
||||
import { CopilotSettings } from "@/settings/model";
|
||||
import { App, FileSystemAdapter, Platform } from "obsidian";
|
||||
|
||||
/**
|
||||
|
|
@ -36,21 +36,12 @@ export function shouldUseMiyo(settings: CopilotSettings): boolean {
|
|||
}
|
||||
|
||||
/**
|
||||
* Resolve the vault identifier sent to Miyo.
|
||||
*
|
||||
* Uses the user-configured remote vault path only when a remote server URL is also
|
||||
* configured, otherwise falls back to the vault filesystem path or vault name.
|
||||
* Resolve the current vault folder path sent to Miyo as `folder_path`.
|
||||
*
|
||||
* @param app - Obsidian application instance.
|
||||
* @returns Remote vault path when a remote server is configured, vault folder path when available, otherwise vault name.
|
||||
* @returns Current vault filesystem path when available, otherwise vault name.
|
||||
*/
|
||||
export function getMiyoVault(app: App): string {
|
||||
const settings = getSettings();
|
||||
const remoteVaultPath = (settings.miyoRemoteVaultPath || "").trim();
|
||||
const serverUrl = (settings.miyoServerUrl || "").trim();
|
||||
if (remoteVaultPath && serverUrl) {
|
||||
return remoteVaultPath;
|
||||
}
|
||||
export function getMiyoFolderPath(app: App): string {
|
||||
const vaultPath = getVaultBasePath(app);
|
||||
if (vaultPath) {
|
||||
return vaultPath;
|
||||
|
|
@ -59,22 +50,48 @@ export function getMiyoVault(app: App): string {
|
|||
}
|
||||
|
||||
/**
|
||||
* Resolve the vault identifier that would apply given a vault path override, used for UI preview.
|
||||
* Resolve an absolute filesystem path for a vault file so it can be sent to Miyo.
|
||||
*
|
||||
* @param app - Obsidian application instance.
|
||||
* @param vaultPathOverride - The vault path to use (empty string = auto-detect).
|
||||
* @returns Overridden vault path, or auto-detected vault path/name.
|
||||
* @param vaultRelativePath - Vault-relative note path.
|
||||
* @returns Absolute file path when available, otherwise the original path.
|
||||
*/
|
||||
export function resolveMiyoVault(app: App, vaultNameOverride: string): string {
|
||||
const trimmed = vaultNameOverride.trim();
|
||||
if (trimmed) {
|
||||
return trimmed;
|
||||
export function getMiyoAbsolutePath(app: App, vaultRelativePath: string): string {
|
||||
const adapter = app.vault.adapter;
|
||||
if (adapter instanceof FileSystemAdapter) {
|
||||
return adapter.getFullPath(vaultRelativePath);
|
||||
}
|
||||
|
||||
const adapterAny = adapter as unknown as { getFullPath?: (normalizedPath: string) => string };
|
||||
if (typeof adapterAny.getFullPath === "function") {
|
||||
return adapterAny.getFullPath(vaultRelativePath);
|
||||
}
|
||||
|
||||
return vaultRelativePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Miyo file path back to a vault-relative path when it belongs to the current vault.
|
||||
*
|
||||
* @param app - Obsidian application instance.
|
||||
* @param miyoPath - Path returned by Miyo.
|
||||
* @returns Vault-relative path when the file is inside the current vault, otherwise the original path.
|
||||
*/
|
||||
export function getVaultRelativeMiyoPath(app: App, miyoPath: string): string {
|
||||
const vaultPath = getVaultBasePath(app);
|
||||
if (vaultPath) {
|
||||
return vaultPath;
|
||||
if (!vaultPath) {
|
||||
return miyoPath;
|
||||
}
|
||||
return app.vault.getName();
|
||||
|
||||
const normalizedVaultPath = normalizeFilesystemPath(vaultPath);
|
||||
const normalizedMiyoPath = normalizeFilesystemPath(miyoPath);
|
||||
const prefix = `${normalizedVaultPath}/`;
|
||||
|
||||
if (normalizedMiyoPath.startsWith(prefix)) {
|
||||
return normalizedMiyoPath.slice(prefix.length);
|
||||
}
|
||||
|
||||
return miyoPath;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -83,7 +100,7 @@ export function resolveMiyoVault(app: App, vaultNameOverride: string): string {
|
|||
* @param app - Obsidian application instance.
|
||||
* @returns Vault base path or undefined when unavailable.
|
||||
*/
|
||||
function getVaultBasePath(app: App): string | undefined {
|
||||
export function getVaultBasePath(app: App): string | undefined {
|
||||
const adapter = app.vault.adapter;
|
||||
if (adapter instanceof FileSystemAdapter) {
|
||||
return adapter.getBasePath();
|
||||
|
|
@ -98,3 +115,13 @@ function getVaultBasePath(app: App): string | undefined {
|
|||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a filesystem path for prefix comparisons across platforms.
|
||||
*
|
||||
* @param path - Filesystem path.
|
||||
* @returns Normalized path with forward slashes and no trailing slash.
|
||||
*/
|
||||
function normalizeFilesystemPath(path: string): string {
|
||||
return path.replace(/\\/g, "/").replace(/\/+$/, "");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,12 @@ import { TFile } from "obsidian";
|
|||
import { getBacklinkedNotes, getLinkedNotes } from "@/noteUtils";
|
||||
import { findRelevantNotes } from "@/search/findRelevantNotes";
|
||||
import { MiyoClient } from "@/miyo/MiyoClient";
|
||||
import { getMiyoVault, shouldUseMiyo } from "@/miyo/miyoUtils";
|
||||
import {
|
||||
getMiyoAbsolutePath,
|
||||
getMiyoFolderPath,
|
||||
getVaultRelativeMiyoPath,
|
||||
shouldUseMiyo,
|
||||
} from "@/miyo/miyoUtils";
|
||||
import { getSettings } from "@/settings/model";
|
||||
import VectorStoreManager from "@/search/vectorStoreManager";
|
||||
|
||||
|
|
@ -47,7 +52,9 @@ jest.mock("@/miyo/MiyoClient", () => ({
|
|||
}));
|
||||
|
||||
jest.mock("@/miyo/miyoUtils", () => ({
|
||||
getMiyoVault: jest.fn(),
|
||||
getMiyoFolderPath: jest.fn(),
|
||||
getMiyoAbsolutePath: jest.fn((_: unknown, path: string) => `/vault/${path}`),
|
||||
getVaultRelativeMiyoPath: jest.fn((_: unknown, path: string) => path.replace("/vault/", "")),
|
||||
getMiyoCustomUrl: jest.fn().mockReturnValue(""),
|
||||
shouldUseMiyo: jest.fn(),
|
||||
}));
|
||||
|
|
@ -76,7 +83,15 @@ describe("findRelevantNotes", () => {
|
|||
const mockedGetBacklinkedNotes = getBacklinkedNotes as jest.MockedFunction<
|
||||
typeof getBacklinkedNotes
|
||||
>;
|
||||
const mockedGetMiyoSourceId = getMiyoVault as jest.MockedFunction<typeof getMiyoVault>;
|
||||
const mockedGetMiyoFolderPath = getMiyoFolderPath as jest.MockedFunction<
|
||||
typeof getMiyoFolderPath
|
||||
>;
|
||||
const mockedGetMiyoAbsolutePath = getMiyoAbsolutePath as jest.MockedFunction<
|
||||
typeof getMiyoAbsolutePath
|
||||
>;
|
||||
const mockedGetVaultRelativeMiyoPath = getVaultRelativeMiyoPath as jest.MockedFunction<
|
||||
typeof getVaultRelativeMiyoPath
|
||||
>;
|
||||
const mockedVectorStoreManager = VectorStoreManager as unknown as {
|
||||
getInstance: () => {
|
||||
getDocumentsByPath: jest.Mock;
|
||||
|
|
@ -98,7 +113,11 @@ describe("findRelevantNotes", () => {
|
|||
} as any);
|
||||
mockedGetLinkedNotes.mockReturnValue([]);
|
||||
mockedGetBacklinkedNotes.mockReturnValue([]);
|
||||
mockedGetMiyoSourceId.mockReturnValue("test-source");
|
||||
mockedGetMiyoFolderPath.mockReturnValue("/vault");
|
||||
mockedGetMiyoAbsolutePath.mockImplementation((_: unknown, path: string) => `/vault/${path}`);
|
||||
mockedGetVaultRelativeMiyoPath.mockImplementation((_: unknown, path: string) =>
|
||||
path.replace("/vault/", "")
|
||||
);
|
||||
|
||||
const source = createMarkdownFile("source.md");
|
||||
const first = createMarkdownFile("first.md");
|
||||
|
|
@ -198,10 +217,10 @@ describe("findRelevantNotes", () => {
|
|||
mockResolveBaseUrl.mockResolvedValue("http://127.0.0.1:8742");
|
||||
mockSearchRelated.mockResolvedValue({
|
||||
results: [
|
||||
{ id: "self", path: "source.md", score: 0.99, chunk_text: "self" },
|
||||
{ id: "a-1", path: "alpha.md", score: 0.45, chunk_text: "alpha1" },
|
||||
{ id: "b-1", path: "beta.md", score: 0.88, chunk_text: "beta" },
|
||||
{ id: "a-2", path: "alpha.md", score: 0.6, chunk_text: "alpha2" },
|
||||
{ id: "self", path: "/vault/source.md", score: 0.99, chunk_text: "self" },
|
||||
{ id: "a-1", path: "/vault/alpha.md", score: 0.45, chunk_text: "alpha1" },
|
||||
{ id: "b-1", path: "/vault/beta.md", score: 0.88, chunk_text: "beta" },
|
||||
{ id: "a-2", path: "/vault/alpha.md", score: 0.6, chunk_text: "alpha2" },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -214,8 +233,8 @@ describe("findRelevantNotes", () => {
|
|||
expect(mockGetDb).not.toHaveBeenCalled();
|
||||
expect(mockGetDocumentsByPath).not.toHaveBeenCalled();
|
||||
expect(mockSearchRelated).toHaveBeenCalledTimes(1);
|
||||
expect(mockSearchRelated).toHaveBeenCalledWith("http://127.0.0.1:8742", "source.md", {
|
||||
vault: "test-source",
|
||||
expect(mockSearchRelated).toHaveBeenCalledWith("http://127.0.0.1:8742", "/vault/source.md", {
|
||||
folderPath: "/vault",
|
||||
limit: 20,
|
||||
});
|
||||
});
|
||||
|
|
@ -235,8 +254,8 @@ describe("findRelevantNotes", () => {
|
|||
mockResolveBaseUrl.mockResolvedValue("http://127.0.0.1:8742");
|
||||
mockSearchRelated.mockResolvedValue({
|
||||
results: [
|
||||
{ id: "a-1", path: "alpha.md", score: 0.75, chunk_text: "alpha chunk" },
|
||||
{ id: "self", path: "source.md", score: 0.99, chunk_text: "self" },
|
||||
{ id: "a-1", path: "/vault/alpha.md", score: 0.75, chunk_text: "alpha chunk" },
|
||||
{ id: "self", path: "/vault/source.md", score: 0.99, chunk_text: "self" },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
import { logInfo, logWarn } from "@/logger";
|
||||
import { MiyoClient } from "@/miyo/MiyoClient";
|
||||
import { getMiyoCustomUrl, getMiyoVault, shouldUseMiyo } from "@/miyo/miyoUtils";
|
||||
import {
|
||||
getMiyoAbsolutePath,
|
||||
getMiyoCustomUrl,
|
||||
getMiyoFolderPath,
|
||||
getVaultRelativeMiyoPath,
|
||||
shouldUseMiyo,
|
||||
} from "@/miyo/miyoUtils";
|
||||
import { getBacklinkedNotes, getLinkedNotes } from "@/noteUtils";
|
||||
import { DBOperations } from "@/search/dbOperations";
|
||||
import type { SemanticIndexDocument } from "@/search/indexBackend/SemanticIndexBackend";
|
||||
|
|
@ -123,24 +129,25 @@ async function calculateSimilarityScoreFromMiyo(filePath: string): Promise<Map<s
|
|||
const settings = getSettings();
|
||||
const miyoClient = new MiyoClient();
|
||||
const baseUrl = await miyoClient.resolveBaseUrl(getMiyoCustomUrl(settings));
|
||||
const vault = getMiyoVault(app);
|
||||
const response = await miyoClient.searchRelated(baseUrl, filePath, {
|
||||
vault,
|
||||
const folderPath = getMiyoFolderPath(app);
|
||||
const response = await miyoClient.searchRelated(baseUrl, getMiyoAbsolutePath(app, filePath), {
|
||||
folderPath,
|
||||
limit: MAX_K,
|
||||
});
|
||||
const similarityScoreMap = new Map<string, number>();
|
||||
const results = response.results || [];
|
||||
|
||||
for (const result of results) {
|
||||
if (result.path === filePath) {
|
||||
const relativePath = getVaultRelativeMiyoPath(app, result.path);
|
||||
if (relativePath === filePath) {
|
||||
continue;
|
||||
}
|
||||
if (typeof result.score !== "number" || Number.isNaN(result.score)) {
|
||||
continue;
|
||||
}
|
||||
const existing = similarityScoreMap.get(result.path);
|
||||
const existing = similarityScoreMap.get(relativePath);
|
||||
if (existing === undefined || result.score > existing) {
|
||||
similarityScoreMap.set(result.path, result.score);
|
||||
similarityScoreMap.set(relativePath, result.score);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
import { Embeddings } from "@langchain/core/embeddings";
|
||||
import { App, Notice } from "obsidian";
|
||||
import { CustomError } from "@/error";
|
||||
import { logError, logInfo, logWarn } from "@/logger";
|
||||
import { MiyoClient, MiyoUpsertDocument } from "@/miyo/MiyoClient";
|
||||
import { getMiyoCustomUrl, getMiyoVault } from "@/miyo/miyoUtils";
|
||||
import { getMatchingPatterns, shouldIndexFile } from "@/search/searchUtils";
|
||||
import { logInfo, logWarn } from "@/logger";
|
||||
import { MiyoClient, MiyoIndexedFileEntry } from "@/miyo/MiyoClient";
|
||||
import {
|
||||
getMiyoAbsolutePath,
|
||||
getMiyoCustomUrl,
|
||||
getMiyoFolderPath,
|
||||
getVaultRelativeMiyoPath,
|
||||
} from "@/miyo/miyoUtils";
|
||||
import type {
|
||||
SemanticIndexBackend,
|
||||
SemanticIndexDocument,
|
||||
|
|
@ -13,10 +16,12 @@ import { getSettings } from "@/settings/model";
|
|||
|
||||
/**
|
||||
* Miyo-backed implementation of the semantic index backend.
|
||||
*
|
||||
* Miyo owns folder registration, chunking, indexing, and file-change tracking.
|
||||
* Copilot only requests scans and reads indexed results.
|
||||
*/
|
||||
export class MiyoIndexBackend implements SemanticIndexBackend {
|
||||
private client: MiyoClient;
|
||||
private filesMissingEmbeddings: Set<string> = new Set();
|
||||
|
||||
/**
|
||||
* Create a new Miyo backend tied to the current Obsidian app instance.
|
||||
|
|
@ -28,28 +33,29 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
}
|
||||
|
||||
/**
|
||||
* Initialize the Miyo backend by ensuring the service is reachable.
|
||||
* Initialize the Miyo backend by validating connectivity and folder access.
|
||||
*
|
||||
* @param _embeddingInstance - Embeddings instance (unused for Miyo initialization).
|
||||
*/
|
||||
public async initialize(_embeddingInstance: Embeddings | undefined): Promise<void> {
|
||||
try {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
await this.client.getStats(baseUrl, this.getSourceId());
|
||||
await this.client.getFolder(baseUrl, this.getFolderPath());
|
||||
} catch (error) {
|
||||
logWarn(`Miyo backend initialization failed: ${error}`);
|
||||
new Notice("Failed to initialize Miyo backend. Check Miyo service discovery or URL.");
|
||||
new Notice(
|
||||
"Failed to initialize Miyo backend. Check Miyo service discovery or folder setup."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all indexed data for the current vault.
|
||||
* Clearing the index is managed directly in Miyo, not by Copilot.
|
||||
*
|
||||
* @param _embeddingInstance - Embeddings instance (unused for Miyo clear).
|
||||
* @param _embeddingInstance - Embeddings instance (unused for Miyo).
|
||||
*/
|
||||
public async clearIndex(_embeddingInstance: Embeddings | undefined): Promise<void> {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
await this.client.clearIndex(baseUrl, this.getSourceId());
|
||||
logWarn("Miyo clearIndex requested from Copilot, but folder lifecycle is managed in Miyo.");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -62,96 +68,66 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
}
|
||||
|
||||
/**
|
||||
* Insert or update a single document.
|
||||
* Copilot no longer upserts individual Miyo documents.
|
||||
*
|
||||
* @param doc - Document to upsert.
|
||||
* @returns The document when upsert succeeds.
|
||||
* @returns Undefined because the write is handled by Miyo scans.
|
||||
*/
|
||||
public async upsert(doc: SemanticIndexDocument): Promise<SemanticIndexDocument | undefined> {
|
||||
const count = await this.upsertBatch([doc]);
|
||||
return count > 0 ? doc : undefined;
|
||||
logInfo(`Skipping direct Miyo upsert for ${doc.path}; Miyo manages indexing itself.`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert or update multiple documents in a batch.
|
||||
* Copilot no longer batches document upserts into Miyo.
|
||||
*
|
||||
* @param docs - Documents to upsert.
|
||||
* @returns Number of documents upserted.
|
||||
* @returns Zero because direct upserts are no longer used.
|
||||
*/
|
||||
public async upsertBatch(docs: SemanticIndexDocument[]): Promise<number> {
|
||||
if (docs.length === 0) {
|
||||
return 0;
|
||||
if (docs.length > 0) {
|
||||
logInfo(`Skipping direct Miyo batch upsert for ${docs.length} documents.`);
|
||||
}
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
const sourceId = this.getSourceId();
|
||||
this.logUpsertBatchRequested(docs, baseUrl, sourceId);
|
||||
const payloadDocs = docs.map((doc) => this.toMiyoDocument(doc));
|
||||
const upserted = await this.client.upsertDocuments(baseUrl, sourceId, payloadDocs);
|
||||
this.logUpsertBatchResult(docs.length, upserted);
|
||||
return upserted;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all documents associated with a file path.
|
||||
* File removal is managed by Miyo's folder watcher and scanner.
|
||||
*
|
||||
* @param path - File path to delete.
|
||||
*/
|
||||
public async removeByPath(path: string): Promise<void> {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
await this.client.deleteByPath(baseUrl, this.getSourceId(), path);
|
||||
logInfo(`Skipping direct Miyo delete for ${path}; Miyo manages file lifecycle itself.`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all indexed file paths.
|
||||
* Return all indexed file paths for the current vault folder.
|
||||
*/
|
||||
public async getIndexedFiles(): Promise<string[]> {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
const sourceId = this.getSourceId();
|
||||
const limit = 200;
|
||||
let offset = 0;
|
||||
let total: number | null = null;
|
||||
const paths = new Set<string>();
|
||||
|
||||
do {
|
||||
const response = await this.client.listFiles(baseUrl, sourceId, offset, limit);
|
||||
const files = response.files ?? [];
|
||||
files.forEach((file) => paths.add(file.path));
|
||||
// Read total only from the first response to avoid drift from concurrent indexing.
|
||||
if (total === null) {
|
||||
total = response.total ?? files.length;
|
||||
}
|
||||
offset += files.length;
|
||||
if (files.length === 0) {
|
||||
break;
|
||||
}
|
||||
} while (offset < total);
|
||||
|
||||
return Array.from(paths).sort();
|
||||
const files = await this.getAllIndexedFiles();
|
||||
return Array.from(new Set(files.map((file) => this.toVaultPath(file.path)))).sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the latest modified time across indexed files.
|
||||
*/
|
||||
public async getLatestFileMtime(): Promise<number> {
|
||||
const stats = await this.getStatsSafely();
|
||||
return stats?.latest_mtime ?? 0;
|
||||
const files = await this.getAllIndexedFiles();
|
||||
return files.reduce((latest, file) => Math.max(latest, file.mtime ?? 0), 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true when the index has no documents.
|
||||
*/
|
||||
public async isIndexEmpty(): Promise<boolean> {
|
||||
const stats = await this.getStatsSafely();
|
||||
if (!stats) {
|
||||
return true;
|
||||
}
|
||||
return (stats.total_chunks ?? 0) === 0;
|
||||
const files = await this.getAllIndexedFiles();
|
||||
return files.length === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true when the specified file path has indexed documents.
|
||||
*
|
||||
* @param path - File path to check.
|
||||
* @param path - Vault-relative file path to check.
|
||||
*/
|
||||
public async hasIndex(path: string): Promise<boolean> {
|
||||
const docs = await this.getDocumentsByPath(path);
|
||||
|
|
@ -159,13 +135,18 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
}
|
||||
|
||||
/**
|
||||
* Return all indexed documents for a given file path.
|
||||
* Return all indexed documents for a given vault-relative file path.
|
||||
*
|
||||
* @param path - File path to look up.
|
||||
* @param path - Vault-relative file path to look up.
|
||||
*/
|
||||
public async getDocumentsByPath(path: string): Promise<SemanticIndexDocument[]> {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
const response = await this.client.getDocumentsByPath(baseUrl, this.getSourceId(), path);
|
||||
const absolutePath = getMiyoAbsolutePath(this.app, path);
|
||||
const response = await this.client.getDocumentsByPath(
|
||||
baseUrl,
|
||||
this.getFolderPath(),
|
||||
absolutePath
|
||||
);
|
||||
const docs = response.documents ?? [];
|
||||
return docs.map((doc) => this.fromMiyoDocument(path, doc));
|
||||
}
|
||||
|
|
@ -174,7 +155,7 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
* Detect embedding model changes and indicate whether a rebuild is needed.
|
||||
*
|
||||
* @param _embeddingInstance - Current embeddings instance (unused for Miyo).
|
||||
* @returns True when a rebuild should occur.
|
||||
* @returns False because Miyo controls embeddings.
|
||||
*/
|
||||
public async checkAndHandleEmbeddingModelChange(
|
||||
_embeddingInstance?: Embeddings
|
||||
|
|
@ -197,67 +178,38 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
}
|
||||
|
||||
/**
|
||||
* Remove stale documents that no longer match the vault state.
|
||||
* Garbage collection is managed directly in Miyo.
|
||||
*
|
||||
* @returns Zero because Copilot does not delete Miyo records directly anymore.
|
||||
*/
|
||||
public async garbageCollect(): Promise<number> {
|
||||
try {
|
||||
const files = this.app.vault.getMarkdownFiles();
|
||||
const filePaths = new Set(files.map((file) => file.path));
|
||||
|
||||
const { inclusions, exclusions } = getMatchingPatterns();
|
||||
const allowedPaths = new Set(
|
||||
files
|
||||
.filter((file) => shouldIndexFile(file, inclusions, exclusions))
|
||||
.map((file) => file.path)
|
||||
);
|
||||
|
||||
const indexedPaths = await this.getIndexedFiles();
|
||||
const pathsToRemove = indexedPaths.filter(
|
||||
(path) => !filePaths.has(path) || !allowedPaths.has(path)
|
||||
);
|
||||
|
||||
if (pathsToRemove.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
logInfo("Miyo index: Paths to remove during garbage collection:", pathsToRemove.join(", "));
|
||||
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
const sourceId = this.getSourceId();
|
||||
let deleted = 0;
|
||||
|
||||
for (const path of pathsToRemove) {
|
||||
deleted += await this.client.deleteByPath(baseUrl, sourceId, path);
|
||||
}
|
||||
|
||||
return deleted;
|
||||
} catch (error) {
|
||||
logError("Error garbage collecting the Miyo index:", error);
|
||||
throw new CustomError("Failed to garbage collect the Copilot index.");
|
||||
}
|
||||
logInfo("Skipping Miyo garbage collection; folder lifecycle is managed in Miyo.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a file as missing embeddings.
|
||||
* Track missing embeddings (unused for Miyo).
|
||||
*
|
||||
* @param path - File path to track.
|
||||
* @param _path - File path to track.
|
||||
*/
|
||||
public markFileMissingEmbeddings(path: string): void {
|
||||
this.filesMissingEmbeddings.add(path);
|
||||
public markFileMissingEmbeddings(_path: string): void {
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear tracked files missing embeddings.
|
||||
* Clear tracked files missing embeddings (unused for Miyo).
|
||||
*/
|
||||
public clearFilesMissingEmbeddings(): void {
|
||||
this.filesMissingEmbeddings.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return tracked files missing embeddings.
|
||||
* Return tracked files missing embeddings (unused for Miyo).
|
||||
*
|
||||
* @returns Empty array.
|
||||
*/
|
||||
public getFilesMissingEmbeddings(): string[] {
|
||||
return Array.from(this.filesMissingEmbeddings);
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -283,6 +235,16 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a folder scan in Miyo.
|
||||
*
|
||||
* @param force - Whether to request a forced re-scan.
|
||||
*/
|
||||
public async requestIndexRefresh(force = false): Promise<void> {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
await this.client.scanFolder(baseUrl, this.getFolderPath(), force);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the Miyo base URL using settings and discovery.
|
||||
*
|
||||
|
|
@ -294,90 +256,60 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
}
|
||||
|
||||
/**
|
||||
* Compute the Miyo source id for the current vault.
|
||||
* Return the Miyo folder path for the current vault.
|
||||
*
|
||||
* @returns Source id string.
|
||||
* @returns Folder path string.
|
||||
*/
|
||||
private getSourceId(): string {
|
||||
return getMiyoVault(this.app);
|
||||
private getFolderPath(): string {
|
||||
return getMiyoFolderPath(this.app);
|
||||
}
|
||||
|
||||
/**
|
||||
* Log details about the pending upsert batch for debugging parity with Orama.
|
||||
* Fetch every indexed file entry from Miyo for the current folder.
|
||||
*
|
||||
* @param docs - Documents being upserted.
|
||||
* @param baseUrl - Miyo base URL.
|
||||
* @param sourceId - Vault source id.
|
||||
* @returns Indexed file entries.
|
||||
*/
|
||||
private logUpsertBatchRequested(
|
||||
docs: SemanticIndexDocument[],
|
||||
baseUrl: string,
|
||||
sourceId: string
|
||||
): void {
|
||||
logInfo(`Miyo upsert batch: ${docs.length} documents`, { baseUrl, sourceId });
|
||||
docs.forEach((doc) => {
|
||||
const chunkId = doc.metadata?.chunkId;
|
||||
const chunkSuffix =
|
||||
typeof chunkId === "string" && chunkId.length > 0 ? ` (chunkId: ${chunkId})` : "";
|
||||
logInfo(`Miyo upsert document ${doc.id} @ ${doc.path}${chunkSuffix}`);
|
||||
});
|
||||
private async getAllIndexedFiles(): Promise<MiyoIndexedFileEntry[]> {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
const limit = 200;
|
||||
let offset = 0;
|
||||
let total: number | null = null;
|
||||
const files: MiyoIndexedFileEntry[] = [];
|
||||
|
||||
do {
|
||||
const response = await this.client.listFolderFiles(baseUrl, {
|
||||
folderPath: this.getFolderPath(),
|
||||
offset,
|
||||
limit,
|
||||
});
|
||||
const batch = response.files ?? [];
|
||||
files.push(...batch);
|
||||
if (total === null) {
|
||||
total = response.total ?? batch.length;
|
||||
}
|
||||
offset += batch.length;
|
||||
if (batch.length === 0) {
|
||||
break;
|
||||
}
|
||||
} while (offset < total);
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log the result of a Miyo upsert batch.
|
||||
* Convert a Miyo path to a vault-relative path when possible.
|
||||
*
|
||||
* @param requested - Number of documents requested for upsert.
|
||||
* @param upserted - Number of documents reported by Miyo.
|
||||
* @param path - Miyo path value.
|
||||
* @returns Vault-relative path when the file is inside the current vault.
|
||||
*/
|
||||
private logUpsertBatchResult(requested: number, upserted: number): void {
|
||||
if (upserted === requested) {
|
||||
logInfo(`Miyo upsert batch completed: ${upserted}/${requested} documents`);
|
||||
return;
|
||||
}
|
||||
logWarn(`Miyo upsert batch returned ${upserted}/${requested} documents`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely fetch index stats without throwing.
|
||||
*
|
||||
* @returns Stats response or null when unavailable.
|
||||
*/
|
||||
private async getStatsSafely() {
|
||||
try {
|
||||
const baseUrl = await this.getBaseUrl();
|
||||
return await this.client.getStats(baseUrl, this.getSourceId());
|
||||
} catch (error) {
|
||||
logError(`Failed to fetch Miyo index stats: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Copilot document to a Miyo upsert payload.
|
||||
*
|
||||
* @param doc - Copilot document.
|
||||
* @returns Miyo upsert document payload.
|
||||
*/
|
||||
private toMiyoDocument(doc: SemanticIndexDocument): MiyoUpsertDocument {
|
||||
return {
|
||||
id: doc.id,
|
||||
path: doc.path,
|
||||
title: doc.title,
|
||||
content: doc.content,
|
||||
created_at: doc.created_at,
|
||||
ctime: doc.ctime,
|
||||
mtime: doc.mtime,
|
||||
tags: doc.tags,
|
||||
extension: doc.extension,
|
||||
nchars: doc.nchars,
|
||||
metadata: doc.metadata ?? {},
|
||||
};
|
||||
private toVaultPath(path: string): string {
|
||||
return getVaultRelativeMiyoPath(this.app, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Miyo document response to a Copilot document shape.
|
||||
*
|
||||
* @param fallbackPath - Path used when payload omits it.
|
||||
* @param fallbackPath - Vault-relative fallback path used when payload omits it.
|
||||
* @param doc - Miyo response document.
|
||||
* @returns Copilot document shape.
|
||||
*/
|
||||
|
|
@ -386,23 +318,24 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
doc: {
|
||||
id: string;
|
||||
path?: string;
|
||||
title?: string;
|
||||
title?: string | null;
|
||||
chunk_index?: number;
|
||||
chunk_text?: string;
|
||||
chunk_text?: string | null;
|
||||
metadata?: Record<string, unknown>;
|
||||
embedding_model?: string;
|
||||
embedding_model?: string | null;
|
||||
ctime?: number;
|
||||
mtime?: number;
|
||||
tags?: string[];
|
||||
extension?: string;
|
||||
created_at?: number;
|
||||
created_at?: string | number | null;
|
||||
nchars?: number;
|
||||
}
|
||||
): SemanticIndexDocument {
|
||||
const resolvedPath = doc.path ? this.toVaultPath(doc.path) : fallbackPath;
|
||||
const content = doc.chunk_text ?? "";
|
||||
const metadata = { ...(doc.metadata ?? {}) };
|
||||
if (!metadata.chunkId && doc.chunk_index !== undefined) {
|
||||
metadata.chunkId = `${doc.path || fallbackPath}#${doc.chunk_index}`;
|
||||
metadata.chunkId = `${resolvedPath}#${doc.chunk_index}`;
|
||||
}
|
||||
if (doc.chunk_index !== undefined && metadata.chunkIndex === undefined) {
|
||||
metadata.chunkIndex = doc.chunk_index;
|
||||
|
|
@ -413,9 +346,9 @@ export class MiyoIndexBackend implements SemanticIndexBackend {
|
|||
title: doc.title ?? "",
|
||||
content,
|
||||
embedding: [],
|
||||
path: doc.path ?? fallbackPath,
|
||||
path: resolvedPath,
|
||||
embeddingModel: doc.embedding_model ?? "",
|
||||
created_at: doc.created_at ?? 0,
|
||||
created_at: typeof doc.created_at === "number" ? doc.created_at : 0,
|
||||
ctime: doc.ctime ?? 0,
|
||||
mtime: doc.mtime ?? 0,
|
||||
tags: doc.tags ?? [],
|
||||
|
|
|
|||
|
|
@ -32,7 +32,11 @@ export class IndexEventHandler {
|
|||
* @returns {boolean} True when semantic search indexing should run.
|
||||
*/
|
||||
private shouldHandleEvents(): boolean {
|
||||
return getSettings().enableSemanticSearchV3;
|
||||
if (!getSettings().enableSemanticSearchV3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !(this.indexBackend.isRemoteBackend() && !this.indexBackend.requiresEmbeddings());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { getMiyoVault } from "@/miyo/miyoUtils";
|
||||
import { getMiyoFolderPath } from "@/miyo/miyoUtils";
|
||||
import { MiyoSemanticRetriever } from "@/search/miyo/MiyoSemanticRetriever";
|
||||
import { getSettings } from "@/settings/model";
|
||||
import { RETURN_ALL_LIMIT } from "@/search/v3/SearchCore";
|
||||
|
|
@ -12,7 +12,8 @@ jest.mock("@/settings/model", () => ({
|
|||
getSettings: jest.fn(),
|
||||
}));
|
||||
jest.mock("@/miyo/miyoUtils", () => ({
|
||||
getMiyoVault: jest.fn(),
|
||||
getMiyoFolderPath: jest.fn(),
|
||||
getVaultRelativeMiyoPath: jest.fn((_: unknown, path: string) => path.replace("/vault/", "")),
|
||||
getMiyoCustomUrl: jest.fn().mockReturnValue(""),
|
||||
}));
|
||||
jest.mock("@/miyo/MiyoClient", () => ({
|
||||
|
|
@ -47,7 +48,7 @@ describe("MiyoSemanticRetriever", () => {
|
|||
miyoServerUrl: "http://miyo.local",
|
||||
debug: false,
|
||||
});
|
||||
(getMiyoVault as jest.Mock).mockReturnValue("vault-source");
|
||||
(getMiyoFolderPath as jest.Mock).mockReturnValue("/vault");
|
||||
mockResolveBaseUrl.mockResolvedValue("http://miyo.local");
|
||||
});
|
||||
|
||||
|
|
@ -57,28 +58,28 @@ describe("MiyoSemanticRetriever", () => {
|
|||
{
|
||||
id: "doc-1",
|
||||
score: 0.9,
|
||||
path: "notes/a.md",
|
||||
path: "/vault/notes/a.md",
|
||||
chunk_index: 0,
|
||||
chunk_text: "A chunk",
|
||||
},
|
||||
{
|
||||
id: "doc-1-dup",
|
||||
score: 0.85,
|
||||
path: "notes/a.md",
|
||||
path: "/vault/notes/a.md",
|
||||
chunk_index: 0,
|
||||
chunk_text: "A duplicated chunk",
|
||||
},
|
||||
{
|
||||
id: "doc-2",
|
||||
score: 0.1,
|
||||
path: "notes/b.md",
|
||||
path: "/vault/notes/b.md",
|
||||
chunk_index: 0,
|
||||
chunk_text: "Below threshold chunk",
|
||||
},
|
||||
{
|
||||
id: "doc-3",
|
||||
score: Number.NaN,
|
||||
path: "notes/c.md",
|
||||
path: "/vault/notes/c.md",
|
||||
chunk_index: 1,
|
||||
chunk_text: "NaN score chunk should pass",
|
||||
},
|
||||
|
|
@ -90,7 +91,7 @@ describe("MiyoSemanticRetriever", () => {
|
|||
|
||||
expect(mockSearch).toHaveBeenCalledWith(
|
||||
"http://miyo.local",
|
||||
"vault-source",
|
||||
"/vault",
|
||||
"query with [[notes/a]] mention",
|
||||
10,
|
||||
undefined
|
||||
|
|
@ -117,7 +118,7 @@ describe("MiyoSemanticRetriever", () => {
|
|||
|
||||
expect(mockSearch).toHaveBeenCalledWith(
|
||||
"http://miyo.local",
|
||||
"vault-source",
|
||||
"/vault",
|
||||
"show notes from this week",
|
||||
10,
|
||||
[{ field: "mtime", gte: startTime, lte: endTime }]
|
||||
|
|
@ -137,7 +138,7 @@ describe("MiyoSemanticRetriever", () => {
|
|||
|
||||
expect(mockSearch).toHaveBeenCalledWith(
|
||||
"http://miyo.local",
|
||||
"vault-source",
|
||||
"/vault",
|
||||
"list all notes about ai digests",
|
||||
RETURN_ALL_LIMIT,
|
||||
undefined
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { BaseRetriever } from "@langchain/core/retrievers";
|
|||
import { App } from "obsidian";
|
||||
import { logInfo, logWarn } from "@/logger";
|
||||
import { MiyoClient, MiyoSearchFilter, MiyoSearchResult } from "@/miyo/MiyoClient";
|
||||
import { getMiyoCustomUrl, getMiyoVault } from "@/miyo/miyoUtils";
|
||||
import { getMiyoCustomUrl, getMiyoFolderPath, getVaultRelativeMiyoPath } from "@/miyo/miyoUtils";
|
||||
import { getSettings } from "@/settings/model";
|
||||
import { RETURN_ALL_LIMIT } from "@/search/v3/SearchCore";
|
||||
|
||||
|
|
@ -91,7 +91,7 @@ export class MiyoSemanticRetriever extends BaseRetriever {
|
|||
}
|
||||
const response = await this.client.search(
|
||||
baseUrl,
|
||||
getMiyoVault(this.app),
|
||||
getMiyoFolderPath(this.app),
|
||||
query,
|
||||
limit,
|
||||
filters
|
||||
|
|
@ -140,10 +140,11 @@ export class MiyoSemanticRetriever extends BaseRetriever {
|
|||
* @returns LangChain Document instance.
|
||||
*/
|
||||
private toDocument(result: MiyoSearchResult): Document {
|
||||
const relativePath = getVaultRelativeMiyoPath(this.app, result.path);
|
||||
const metadata = result.metadata ?? {};
|
||||
const chunkId =
|
||||
metadata.chunkId ||
|
||||
(result.chunk_index !== undefined ? `${result.path}#${result.chunk_index}` : undefined);
|
||||
(result.chunk_index !== undefined ? `${relativePath}#${result.chunk_index}` : undefined);
|
||||
|
||||
const score = typeof result.score === "number" ? result.score.toFixed(2) : "?";
|
||||
return new Document({
|
||||
|
|
@ -152,7 +153,7 @@ export class MiyoSemanticRetriever extends BaseRetriever {
|
|||
...metadata,
|
||||
score: result.score,
|
||||
explanation: `miyo ${score}`,
|
||||
path: result.path,
|
||||
path: relativePath,
|
||||
mtime: result.mtime,
|
||||
ctime: result.ctime,
|
||||
title: result.title ?? "",
|
||||
|
|
|
|||
|
|
@ -130,6 +130,12 @@ export default class VectorStoreManager {
|
|||
return 0;
|
||||
}
|
||||
|
||||
if (this.activeBackendKey === "miyo") {
|
||||
await this.miyoBackend.requestIndexRefresh(Boolean(overwrite));
|
||||
notifyIndexChanged();
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (
|
||||
Platform.isMobile &&
|
||||
getSettings().disableIndexOnMobile &&
|
||||
|
|
@ -281,6 +287,9 @@ export default class VectorStoreManager {
|
|||
|
||||
public async reindexFile(file: TFile): Promise<void> {
|
||||
await this.waitForInitialization();
|
||||
if (this.activeBackendKey === "miyo") {
|
||||
return;
|
||||
}
|
||||
await this.indexOps.reindexFile(file);
|
||||
notifyIndexChanged();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ export interface CopilotSettings {
|
|||
selfHostApiKey: string;
|
||||
/** Custom Miyo server URL, e.g. "http://192.168.1.10:8742" (empty = use local service discovery) */
|
||||
miyoServerUrl: string;
|
||||
/** Remote vault path sent to Miyo as the vault's source identifier (overrides auto-detected vault path). Only relevant when a remote Miyo server URL is configured. */
|
||||
/** @deprecated Miyo now uses the current vault path as `folder_path`; preserved only for backwards-compatible settings migration. */
|
||||
miyoRemoteVaultPath: string;
|
||||
/** Which provider to use for self-host web search */
|
||||
selfHostSearchProvider: "firecrawl" | "perplexity";
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { HelpTooltip } from "@/components/ui/help-tooltip";
|
|||
import { SettingItem } from "@/components/ui/setting-item";
|
||||
import { DEFAULT_SETTINGS } from "@/constants";
|
||||
import { MiyoClient } from "@/miyo/MiyoClient";
|
||||
import { getMiyoCustomUrl, resolveMiyoVault } from "@/miyo/miyoUtils";
|
||||
import { getMiyoCustomUrl, getMiyoFolderPath } from "@/miyo/miyoUtils";
|
||||
import { useIsSelfHostEligible, validateSelfHostMode } from "@/plusUtils";
|
||||
import { updateSetting, useSettingsValue } from "@/settings/model";
|
||||
import { Notice } from "obsidian";
|
||||
|
|
@ -86,8 +86,8 @@ export const CopilotPlusSettings: React.FC = () => {
|
|||
new ConfirmModal(
|
||||
app,
|
||||
confirmChange,
|
||||
`Enabling Miyo Search will refresh your vault index to store data in Miyo. Embedding Batch Size will be reset to the default (${DEFAULT_SETTINGS.embeddingBatchSize}) for local stability. If your hardware is strong, you can increase this later in QA settings. Continue?`,
|
||||
"Refresh Index"
|
||||
`Enabling Miyo Search will use your current vault path as the Miyo folder path and request a scan from Miyo. Make sure this folder is already registered in Miyo. Embedding Batch Size will be reset to the default (${DEFAULT_SETTINGS.embeddingBatchSize}) for local stability. Continue?`,
|
||||
"Request Miyo Scan"
|
||||
).open();
|
||||
};
|
||||
|
||||
|
|
@ -227,24 +227,10 @@ export const CopilotPlusSettings: React.FC = () => {
|
|||
onChange={(value) => updateSetting("miyoServerUrl", value)}
|
||||
/>
|
||||
|
||||
<SettingItem
|
||||
type="text"
|
||||
title="Remote Vault Path (Optional)"
|
||||
description={
|
||||
<span>
|
||||
Leave blank when accessing Miyo locally. Set this only when using a remote
|
||||
Miyo server to override the vault identifier. Must match exactly across all
|
||||
devices sharing the same remote Miyo instance.
|
||||
</span>
|
||||
}
|
||||
value={settings.miyoRemoteVaultPath || ""}
|
||||
onChange={(value) => updateSetting("miyoRemoteVaultPath", value)}
|
||||
/>
|
||||
|
||||
<SettingItem
|
||||
type="switch"
|
||||
title="Enable Miyo"
|
||||
description="Use Miyo as your local search, PDF parsing, and context hub. Enabling this will prompt a one-time index refresh."
|
||||
description="Use Miyo as your local search, PDF parsing, and context hub. Copilot will send the current vault path to Miyo and can request scans, but folder registration is managed in Miyo."
|
||||
checked={settings.enableMiyo}
|
||||
onCheckedChange={handleMiyoSearchToggle}
|
||||
disabled={isValidatingSelfHost}
|
||||
|
|
@ -252,14 +238,9 @@ export const CopilotPlusSettings: React.FC = () => {
|
|||
|
||||
{settings.enableMiyo && (
|
||||
<div className="tw-text-xs tw-text-muted">
|
||||
Vault path:{" "}
|
||||
Folder path sent to Miyo:{" "}
|
||||
<span className="tw-font-medium tw-text-normal">
|
||||
{resolveMiyoVault(
|
||||
app,
|
||||
(settings.miyoServerUrl || "").trim()
|
||||
? settings.miyoRemoteVaultPath || ""
|
||||
: ""
|
||||
)}
|
||||
{getMiyoFolderPath(app)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { TEXT_WEIGHT } from "@/constants";
|
|||
import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
|
||||
import { hasSelfHostSearchKey, selfHostWebSearch } from "@/LLMProviders/selfHostServices";
|
||||
import { logInfo } from "@/logger";
|
||||
import { shouldUseMiyo } from "@/miyo/miyoUtils";
|
||||
import { isSelfHostModeValid } from "@/plusUtils";
|
||||
import { RetrieverFactory } from "@/search/RetrieverFactory";
|
||||
import { getSettings } from "@/settings/model";
|
||||
|
|
@ -509,11 +510,17 @@ const indexTool = createLangChainTool({
|
|||
try {
|
||||
const VectorStoreManager = (await import("@/search/vectorStoreManager")).default;
|
||||
const count = await VectorStoreManager.getInstance().indexVaultToVectorStore();
|
||||
const indexResultPrompt = `Semantic search index refreshed with ${count} documents.\n`;
|
||||
const usingMiyo = shouldUseMiyo(settings);
|
||||
const indexResultPrompt = usingMiyo
|
||||
? "Requested a Miyo folder scan for this vault.\n"
|
||||
: `Semantic search index refreshed with ${count} documents.\n`;
|
||||
return {
|
||||
success: true,
|
||||
message:
|
||||
indexResultPrompt + `Semantic search index has been refreshed with ${count} documents.`,
|
||||
message: usingMiyo
|
||||
? indexResultPrompt +
|
||||
"Miyo will handle chunking and indexing for the registered folder."
|
||||
: indexResultPrompt +
|
||||
`Semantic search index has been refreshed with ${count} documents.`,
|
||||
documentCount: count,
|
||||
};
|
||||
} catch (error: any) {
|
||||
|
|
|
|||
Loading…
Reference in a new issue