mirror of
https://github.com/aaronsb/obsidian-mcp-plugin.git
synced 2026-07-22 06:45:14 +00:00
Measured against two vaults with a paired A/B harness (plugin tools vs. a shell with ripgrep, same question, isolated agents). On a graph-shaped question over a densely linked vault — the plugin's home field — it LOST: 90.3k tokens vs 70.1k, 20 calls vs 16, 3 dead ends vs 2. Not because the graph failed. Because the plugin's own output made the graph unusable, and the agent spent the difference working around it. Paths were not usable as identifiers vault.search elided long paths to `first/.../last`, and vault.list printed only the basename. Both destroy the one field an agent must have verbatim: the value it feeds to the very next call. An agent that joined the listed directory to the name it was given got a path that does not exist and a hard "File not found". This is the mechanism behind the repeated-search behaviour we keep seeing. Scan-then-follow requires handing a search hit to graph.neighbors; if the hit's path is elided, that handoff is impossible, so the only move left that still works is another search. The tool was training the behaviour. Paths are now emitted verbatim everywhere, and formatPath — a helper whose sole purpose was to damage them — is deleted. vault.fragments ignored its own `path` parameter `path` was read only as a fallback *query* string, never as a scope, so asking for fragments from one note returned passages from several, top-ranked hit first. An agent reasoning "these are the key passages in the note I named" would attribute content to the wrong file. Now scoped, over-fetching candidates first so scoping does not silently return nothing when other notes outrank the target. graph.statistics threw on the vault-wide call No sourcePath is returned for the vault-wide shape; the formatter read it unconditionally, threw, and fell back to raw JSON behind a "Formatter error" banner. Now formatted — and it reports link density, because that is the number that tells the caller whether following links will beat searching. Steering, since the affordance was invisible Both experiment arms independently concluded the graph was the strongest signal available and both nearly missed it: "graph.neighbors was the MVP and I nearly didn't reach for it"; and from the ripgrep arm, "I had a graph and was forced to walk it as a pile of bytes." Search results now name graph.neighbors with the top hit's path filled in, and the vault/graph tool descriptions teach the pattern: a couple of broad scans, then follow links — rather than many narrow searches that re-run the same ranking. vault.read printed the frontmatter twice A truncated "## Frontmatter" summary above a body that already opens with the raw block. Suppressed when the body carries it; retained for fragment reads, which do not. Fragment strategy `semantic` renamed to `structure` It cuts on headings and paragraphs. It has never been embedding or vector search, but "semantic" in a retrieval slot reads as exactly that to any model trained on the last decade of the literature — so callers selected it expecting matching this index does not do. `semantic` still works as a deprecated alias. All four strategies are now described; previously the enum shipped with no gloss at all and the agent chose blind. Tests: 415 passing. New suites cover path fidelity (including the fabricated path an agent would construct), fragment scoping, the vault-wide stats shape, and the frontmatter duplication.
108 lines
3.8 KiB
TypeScript
108 lines
3.8 KiB
TypeScript
/**
|
|
* vault.fragments — the `path` parameter must scope the search.
|
|
*
|
|
* Measured failure: asking for fragments from one note returned "fragments across 3
|
|
* files", with the top-scoring passage coming from a different note entirely. The `path`
|
|
* param was only read as a fallback *query* string, never as a scope, so an agent that
|
|
* reasoned "these are the key passages in the note I named" would attribute content to
|
|
* the wrong file. That is a correctness trap, not a presentation nit.
|
|
*/
|
|
import { UniversalFragmentRetriever } from '../src/indexing/fragment-retriever';
|
|
|
|
const TARGET = 'scaled-sandwich/organizational-patterns.md';
|
|
const OTHER = 'scaled-sandwich/README.md';
|
|
|
|
function retrieverWithTwoDocs(): UniversalFragmentRetriever {
|
|
const retriever = new UniversalFragmentRetriever();
|
|
|
|
retriever.indexDocument(
|
|
`file:${TARGET}`,
|
|
TARGET,
|
|
[
|
|
'# Organizational patterns',
|
|
'Kanban WIP limits prevent overwhelming the human reviewer, maintaining quality.',
|
|
'Pull systems prevent overload. Span of control caps the reviewer at seven items.',
|
|
'Context switching is expensive, so batch similar decisions together.'
|
|
].join('\n\n')
|
|
);
|
|
|
|
retriever.indexDocument(
|
|
`file:${OTHER}`,
|
|
OTHER,
|
|
[
|
|
'# Readme',
|
|
'Pull, do not push: use WIP limits to prevent overwhelming reviewers.',
|
|
'Trust is earned through progressive autonomy and Kanban flow management.'
|
|
].join('\n\n')
|
|
);
|
|
|
|
return retriever;
|
|
}
|
|
|
|
describe('fragment retrieval scoping', () => {
|
|
// 'adaptive' is pinned in the spanning pair below rather than relying on 'auto':
|
|
// auto picks a strategy per query, and on a corpus this small it may happen to return
|
|
// hits from one document, which would make the control vacuously pass.
|
|
it('should return fragments from every indexed document when unscoped', () => {
|
|
const retriever = retrieverWithTwoDocs();
|
|
|
|
const response = retriever.retrieveFragments('kanban WIP limits reviewer', {
|
|
maxFragments: 10,
|
|
strategy: 'adaptive'
|
|
});
|
|
const paths = new Set((response.result ?? []).map(f => f.docPath));
|
|
|
|
expect(paths.has(TARGET)).toBe(true);
|
|
expect(paths.has(OTHER)).toBe(true);
|
|
});
|
|
|
|
it('should return fragments only from the scoped document', () => {
|
|
const retriever = retrieverWithTwoDocs();
|
|
|
|
const response = retriever.retrieveFragments('kanban WIP limits reviewer', {
|
|
maxFragments: 10,
|
|
strategy: 'adaptive',
|
|
scopePath: TARGET
|
|
});
|
|
const paths = new Set((response.result ?? []).map(f => f.docPath));
|
|
|
|
expect(paths.has(OTHER)).toBe(false);
|
|
expect([...paths]).toEqual([TARGET]);
|
|
});
|
|
|
|
it('should scope under the default auto strategy too', () => {
|
|
const retriever = retrieverWithTwoDocs();
|
|
|
|
const response = retriever.retrieveFragments('kanban WIP limits reviewer', {
|
|
maxFragments: 10,
|
|
scopePath: TARGET
|
|
});
|
|
|
|
expect((response.result ?? []).every(f => f.docPath === TARGET)).toBe(true);
|
|
});
|
|
|
|
it('should still find fragments in the scoped document when another document ranks higher', () => {
|
|
const retriever = retrieverWithTwoDocs();
|
|
|
|
// Scoping must not simply filter the unscoped top-N — if the other document's
|
|
// passages outrank the target's, filtering afterwards would return nothing.
|
|
const response = retriever.retrieveFragments('pull systems prevent overload', {
|
|
maxFragments: 1,
|
|
scopePath: TARGET
|
|
});
|
|
|
|
expect(response.result?.length).toBeGreaterThan(0);
|
|
expect(response.result?.[0].docPath).toBe(TARGET);
|
|
});
|
|
|
|
it('should return no fragments when scoped to a document that was never indexed', () => {
|
|
const retriever = retrieverWithTwoDocs();
|
|
|
|
const response = retriever.retrieveFragments('kanban', {
|
|
maxFragments: 5,
|
|
scopePath: 'nowhere/missing.md'
|
|
});
|
|
|
|
expect(response.result ?? []).toEqual([]);
|
|
});
|
|
});
|