mirror of
https://github.com/andy-stack/vaultkeeper-ai.git
synced 2026-07-22 06:42:03 +00:00
Add page number tracking to search snippets
- Add pageNumber field to ISearchSnippet and IPageText interfaces - Update extractSnippets to process content as paginated text - Switch PDF reading to use readPDF helper for page extraction - Update search results to include page numbers in snippets - Add page number assertions to VaultService tests
This commit is contained in:
parent
051abbe43b
commit
4d72bba087
6 changed files with 584 additions and 26 deletions
30
Helpers/PDFHelper.ts
Normal file
30
Helpers/PDFHelper.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import { extractText, getDocumentProxy } from 'unpdf';
|
||||
import type { IPageText } from './SearchTypes';
|
||||
import { Exception } from './Exception';
|
||||
|
||||
export async function readPDF(arrayBuffer: ArrayBuffer): Promise<IPageText[]> {
|
||||
try {
|
||||
const pdf = await getDocumentProxy(new Uint8Array(arrayBuffer));
|
||||
const pages = (await extractText(pdf, { mergePages: false })).text;
|
||||
|
||||
const pageTexts: IPageText[] = pages.map((pageText, index) => ({
|
||||
text: pageText,
|
||||
pageNumber: index + 1
|
||||
}));
|
||||
|
||||
return pageTexts;
|
||||
} catch (error) {
|
||||
/** PDF.js error types (from underlying pdfjs-dist library):
|
||||
* - InvalidPDFException: Invalid or corrupted PDF structure
|
||||
* - PasswordException: PDF requires a password
|
||||
* - FormatError: PDF format error
|
||||
* - UnexpectedResponseException: Unexpected server response
|
||||
* - AbortException: Operation was aborted
|
||||
* - UnknownErrorException: Unknown error occurred **/
|
||||
|
||||
if (error instanceof Error && error.name === 'PasswordException') {
|
||||
return [{ text: "PDF is password protected!", pageNumber: 1 }] as IPageText[];
|
||||
}
|
||||
return [{ text: `Failed to read PDF: ${Exception.messageFrom(error)}`, pageNumber: 1 }] as IPageText[];
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,13 @@
|
|||
import { TFile } from "obsidian";
|
||||
|
||||
/**
|
||||
* Represents text content read from a file with the page number
|
||||
*/
|
||||
export interface IPageText {
|
||||
text: string;
|
||||
pageNumber: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a single snippet of matched content from a file
|
||||
*/
|
||||
|
|
@ -7,6 +15,7 @@ export interface ISearchSnippet {
|
|||
text: string;
|
||||
matchIndex: number;
|
||||
matchLength: number;
|
||||
pageNumber: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ export class AIFunctionService {
|
|||
path: match.file.path,
|
||||
snippets: match.snippets.map((snippet) => ({
|
||||
text: snippet.text,
|
||||
pageNumber: snippet.pageNumber,
|
||||
matchPosition: snippet.matchIndex
|
||||
}))
|
||||
}))
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type VaultkeeperAIPlugin from "main";
|
|||
import { Path } from "Enums/Path";
|
||||
import { randomSample, shuffleArray } from "Helpers/Helpers";
|
||||
import { StringTools } from "Helpers/StringTools";
|
||||
import type { ISearchMatch, ISearchSnippet } from "../Helpers/SearchTypes";
|
||||
import type { IPageText, ISearchMatch, ISearchSnippet } from "../Helpers/SearchTypes";
|
||||
import type { SanitiserService } from "./SanitiserService";
|
||||
import { FileEvent } from "Enums/FileEvent";
|
||||
import type { SettingsService } from "./SettingsService";
|
||||
|
|
@ -16,8 +16,8 @@ import * as path from "path-browserify";
|
|||
import { Event } from "Enums/Event";
|
||||
import { AbortService } from "./AbortService";
|
||||
import { AIFunctionResponse } from "AIClasses/FunctionDefinitions/AIFunctionResponse";
|
||||
import { extractText } from 'unpdf';
|
||||
import { FileType, isBinaryFile, isFileType } from "Enums/FileType";
|
||||
import { readPDF } from "Helpers/PDFHelper";
|
||||
|
||||
interface IFileEventArgs {
|
||||
oldPath: string;
|
||||
|
|
@ -326,9 +326,9 @@ export class VaultService {
|
|||
let content;
|
||||
if (isFileType(file.extension.toLocaleLowerCase(), FileType.PDF)) {
|
||||
const arrayBuffer = await this.vault.readBinary(file);
|
||||
content = (await extractText(new Uint8Array(arrayBuffer), { mergePages: true })).text;
|
||||
content = await readPDF(arrayBuffer);
|
||||
} else {
|
||||
content = await this.vault.cachedRead(file);
|
||||
content = [{ text: await this.vault.cachedRead(file), pageNumber: 1 }] as IPageText[];
|
||||
}
|
||||
|
||||
const snippets = this.extractSnippets(content, regex);
|
||||
|
|
@ -423,32 +423,34 @@ export class VaultService {
|
|||
}
|
||||
}
|
||||
|
||||
private extractSnippets(content: string, regex: RegExp): ISearchSnippet[] {
|
||||
const matchPositions: { matchIndex: number; matchLength: number }[] = [];
|
||||
private extractSnippets(pages: IPageText[], regex: RegExp): ISearchSnippet[] {
|
||||
const allSnippets: ISearchSnippet[] = [];
|
||||
|
||||
let match: RegExpExecArray | null;
|
||||
for (const page of pages) {
|
||||
const matchPositions: { matchIndex: number; matchLength: number }[] = [];
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
// First pass: collect all match positions without extracting text
|
||||
while ((match = regex.exec(content)) !== null) {
|
||||
matchPositions.push({
|
||||
matchIndex: match.index,
|
||||
matchLength: match[0].length
|
||||
});
|
||||
// First pass: collect all match positions without extracting text
|
||||
while ((match = regex.exec(page.text)) !== null) {
|
||||
matchPositions.push({
|
||||
matchIndex: match.index,
|
||||
matchLength: match[0].length
|
||||
});
|
||||
}
|
||||
|
||||
regex.lastIndex = 0;
|
||||
|
||||
if (matchPositions.length > 0) {
|
||||
// Second pass: merge overlapping positions and extract text only once per snippet
|
||||
const pageSnippets = this.mergeOverlappingSnippets(matchPositions, page.text, page.pageNumber);
|
||||
allSnippets.push(...pageSnippets);
|
||||
}
|
||||
}
|
||||
|
||||
regex.lastIndex = 0;
|
||||
|
||||
if (matchPositions.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Second pass: merge overlapping positions and extract text only once per snippet
|
||||
return this.mergeOverlappingSnippets(matchPositions, content);
|
||||
return allSnippets;
|
||||
}
|
||||
|
||||
private mergeOverlappingSnippets(
|
||||
matchPositions: { matchIndex: number; matchLength: number }[],
|
||||
content: string
|
||||
private mergeOverlappingSnippets(matchPositions: { matchIndex: number; matchLength: number }[], content: string, pageNumber: number
|
||||
): ISearchSnippet[] {
|
||||
if (matchPositions.length === 0) return [];
|
||||
|
||||
|
|
@ -479,7 +481,8 @@ export class VaultService {
|
|||
merged.push({
|
||||
text: content.substring(snippetStart, snippetEnd),
|
||||
matchIndex: current.matchIndex,
|
||||
matchLength: current.matchLength
|
||||
matchLength: current.matchLength,
|
||||
pageNumber: pageNumber
|
||||
});
|
||||
|
||||
current = next;
|
||||
|
|
@ -493,7 +496,8 @@ export class VaultService {
|
|||
merged.push({
|
||||
text: content.substring(snippetStart, snippetEnd),
|
||||
matchIndex: current.matchIndex,
|
||||
matchLength: current.matchLength
|
||||
matchLength: current.matchLength,
|
||||
pageNumber: pageNumber
|
||||
});
|
||||
|
||||
return merged;
|
||||
|
|
|
|||
510
__tests__/Services/VaultService.pdf.test.ts
Normal file
510
__tests__/Services/VaultService.pdf.test.ts
Normal file
|
|
@ -0,0 +1,510 @@
|
|||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { VaultService } from '../../Services/VaultService';
|
||||
import { TFile, TFolder, FileManager } from 'obsidian';
|
||||
import { Path } from '../../Enums/Path';
|
||||
import { RegisterSingleton, DeregisterAllServices } from '../../Services/DependencyService';
|
||||
import { Services } from '../../Services/Services';
|
||||
import { SanitiserService } from '../../Services/SanitiserService';
|
||||
import { SettingsService, IVaultkeeperAISettings } from '../../Services/SettingsService';
|
||||
import { AIProviderModel } from '../../Enums/ApiProvider';
|
||||
import { Exception } from '../../Helpers/Exception';
|
||||
import * as PDFHelper from '../../Helpers/PDFHelper';
|
||||
import type { IPageText } from '../../Helpers/SearchTypes';
|
||||
|
||||
/**
|
||||
* PDF-SPECIFIC TESTS FOR VAULTSERVICE
|
||||
*
|
||||
* These tests focus on PDF file handling, including:
|
||||
* - Multi-page PDF search
|
||||
* - Page number tracking
|
||||
* - PDF error handling
|
||||
*/
|
||||
|
||||
// Create mock instances
|
||||
const mockVault = {
|
||||
getMarkdownFiles: vi.fn(),
|
||||
getAbstractFileByPath: vi.fn(),
|
||||
read: vi.fn(),
|
||||
cachedRead: vi.fn(),
|
||||
readBinary: vi.fn(),
|
||||
create: vi.fn(),
|
||||
process: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
trash: vi.fn(),
|
||||
createFolder: vi.fn(),
|
||||
getFiles: vi.fn(),
|
||||
getAllFolders: vi.fn(),
|
||||
on: vi.fn(),
|
||||
adapter: {
|
||||
exists: vi.fn()
|
||||
}
|
||||
};
|
||||
|
||||
const mockFileManager = {
|
||||
renameFile: vi.fn(),
|
||||
trashFile: vi.fn()
|
||||
} as unknown as FileManager & {
|
||||
renameFile: ReturnType<typeof vi.fn>;
|
||||
trashFile: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
const mockSettings: IVaultkeeperAISettings = {
|
||||
firstTimeStart: false,
|
||||
model: AIProviderModel.ClaudeSonnet_4_5,
|
||||
apiKeys: {
|
||||
claude: 'test-claude-key',
|
||||
openai: 'test-openai-key',
|
||||
gemini: 'test-gemini-key'
|
||||
},
|
||||
exclusions: [],
|
||||
userInstruction: '',
|
||||
searchResultsLimit: 15,
|
||||
snippetSizeLimit: 300
|
||||
};
|
||||
|
||||
const mockPlugin = {
|
||||
app: {
|
||||
vault: mockVault,
|
||||
fileManager: mockFileManager
|
||||
},
|
||||
saveData: vi.fn().mockResolvedValue(undefined),
|
||||
registerEvent: vi.fn()
|
||||
};
|
||||
|
||||
// Helper to create mock PDF file
|
||||
function createMockPDFFile(path: string): TFile {
|
||||
const name = path.split('/').pop() || '';
|
||||
const basename = name.split('.')[0];
|
||||
const file = new TFile();
|
||||
file.path = path;
|
||||
file.name = name;
|
||||
file.basename = basename;
|
||||
file.extension = 'pdf';
|
||||
file.stat = { ctime: Date.now(), mtime: Date.now(), size: 1000 };
|
||||
file.parent = null;
|
||||
file.vault = mockVault as any;
|
||||
return file;
|
||||
}
|
||||
|
||||
// Helper to create mock markdown file
|
||||
function createMockFile(path: string): TFile {
|
||||
const name = path.split('/').pop() || '';
|
||||
const basename = name.split('.')[0];
|
||||
const file = new TFile();
|
||||
file.path = path;
|
||||
file.name = name;
|
||||
file.basename = basename;
|
||||
file.extension = 'md';
|
||||
file.stat = { ctime: Date.now(), mtime: Date.now(), size: 100 };
|
||||
file.parent = null;
|
||||
file.vault = mockVault as any;
|
||||
return file;
|
||||
}
|
||||
|
||||
// Helper to create mock TFolder
|
||||
function createMockFolder(path: string, children: TFile[] = []): TFolder {
|
||||
const name = path.split('/').pop() || '';
|
||||
const folder = new TFolder();
|
||||
folder.path = path;
|
||||
folder.name = name;
|
||||
folder.children = children;
|
||||
folder.parent = null;
|
||||
folder.vault = mockVault as any;
|
||||
return folder;
|
||||
}
|
||||
|
||||
describe('VaultService - PDF Tests', () => {
|
||||
let vaultService: VaultService;
|
||||
let settingsService: SettingsService;
|
||||
let mockDiffService: any;
|
||||
let readPDFSpy: any;
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset all mocks
|
||||
vi.clearAllMocks();
|
||||
|
||||
// Reset settings to defaults
|
||||
mockSettings.exclusions = [];
|
||||
mockSettings.searchResultsLimit = 15;
|
||||
mockSettings.snippetSizeLimit = 300;
|
||||
|
||||
// Set default mock for adapter.exists
|
||||
mockVault.adapter.exists.mockResolvedValue(false);
|
||||
|
||||
// Mock console.error to prevent noise in tests
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {});
|
||||
|
||||
// Mock Exception.log
|
||||
vi.spyOn(Exception, 'log').mockImplementation(() => {});
|
||||
|
||||
// Register real dependencies in DependencyService
|
||||
RegisterSingleton(Services.VaultkeeperAIPlugin, mockPlugin as any);
|
||||
RegisterSingleton(Services.SanitiserService, new SanitiserService());
|
||||
|
||||
// Mock EventService and DiffService
|
||||
const mockEventService = { trigger: vi.fn(), on: vi.fn(), off: vi.fn() };
|
||||
mockDiffService = {
|
||||
requestDiff: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
applyPatch: vi.fn(),
|
||||
onAccept: vi.fn(),
|
||||
onReject: vi.fn(),
|
||||
onSuggest: vi.fn()
|
||||
};
|
||||
RegisterSingleton(Services.EventService, mockEventService as any);
|
||||
RegisterSingleton(Services.DiffService, mockDiffService as any);
|
||||
|
||||
// Create and register SettingsService
|
||||
settingsService = new SettingsService(mockSettings);
|
||||
RegisterSingleton(Services.SettingsService, settingsService);
|
||||
|
||||
// Spy on readPDF function
|
||||
readPDFSpy = vi.spyOn(PDFHelper, 'readPDF');
|
||||
|
||||
// Create a fresh instance
|
||||
vaultService = new VaultService();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// Clear singleton registry to prevent memory leaks
|
||||
DeregisterAllServices();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('Multi-page PDF search', () => {
|
||||
it('should find matches across multiple pages in a PDF', async () => {
|
||||
const pdfFile = createMockPDFFile('document.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
// Mock PDF with 3 pages, search term appears on pages 1 and 3
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'First page contains the search term', pageNumber: 1 },
|
||||
{ text: 'Second page has different content', pageNumber: 2 },
|
||||
{ text: 'Third page also has the search term', pageNumber: 3 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('search');
|
||||
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
const match = results.find(r => r.file.path === 'document.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets.length).toBe(2); // Two matches across two pages
|
||||
|
||||
// Verify page numbers are correct
|
||||
const pageNumbers = match!.snippets.map(s => s.pageNumber).sort();
|
||||
expect(pageNumbers).toEqual([1, 3]);
|
||||
});
|
||||
|
||||
it('should track correct page numbers for each snippet', async () => {
|
||||
const pdfFile = createMockPDFFile('report.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'Introduction with keyword', pageNumber: 1 },
|
||||
{ text: 'Chapter 1 mentions keyword twice: keyword and keyword', pageNumber: 2 },
|
||||
{ text: 'Chapter 2 has no matches', pageNumber: 3 },
|
||||
{ text: 'Conclusion includes keyword', pageNumber: 4 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('keyword');
|
||||
|
||||
const match = results.find(r => r.file.path === 'report.pdf');
|
||||
expect(match).toBeDefined();
|
||||
|
||||
// Should have 3 snippets: page 1 (1 match), page 2 (merged into 1 snippet), page 4 (1 match)
|
||||
expect(match!.snippets.length).toBeGreaterThan(0);
|
||||
|
||||
// All snippets should have valid page numbers
|
||||
match!.snippets.forEach(snippet => {
|
||||
expect(snippet.pageNumber).toBeGreaterThan(0);
|
||||
expect(snippet.pageNumber).toBeLessThanOrEqual(4);
|
||||
});
|
||||
|
||||
// Check that we have snippets from the correct pages
|
||||
const pageNumbers = match!.snippets.map(s => s.pageNumber);
|
||||
expect(pageNumbers).toContain(1);
|
||||
expect(pageNumbers).toContain(2);
|
||||
expect(pageNumbers).toContain(4);
|
||||
expect(pageNumbers).not.toContain(3); // Page 3 has no matches
|
||||
});
|
||||
|
||||
it('should handle PDF with single page correctly', async () => {
|
||||
const pdfFile = createMockPDFFile('single-page.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'This is a single page document with a match', pageNumber: 1 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('match');
|
||||
|
||||
const match = results.find(r => r.file.path === 'single-page.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle large multi-page PDF efficiently', async () => {
|
||||
const pdfFile = createMockPDFFile('large-document.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
// Simulate a 100-page PDF with matches on specific pages
|
||||
const mockPages: IPageText[] = [];
|
||||
for (let i = 1; i <= 100; i++) {
|
||||
const hasMatch = i % 10 === 0; // Matches on pages 10, 20, 30, etc.
|
||||
mockPages.push({
|
||||
text: hasMatch ? `Page ${i} contains target word` : `Page ${i} content`,
|
||||
pageNumber: i
|
||||
});
|
||||
}
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(1000));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('target');
|
||||
|
||||
const match = results.find(r => r.file.path === 'large-document.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets.length).toBe(10); // 10 matches (pages 10, 20, 30, ..., 100)
|
||||
|
||||
// Verify page numbers are multiples of 10
|
||||
const pageNumbers = match!.snippets.map(s => s.pageNumber).sort((a, b) => (a || 0) - (b || 0));
|
||||
expect(pageNumbers).toEqual([10, 20, 30, 40, 50, 60, 70, 80, 90, 100]);
|
||||
});
|
||||
|
||||
it('should merge overlapping matches within the same page', async () => {
|
||||
const pdfFile = createMockPDFFile('overlapping.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
// Create a page with multiple close matches that should be merged
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'test ' + 'a'.repeat(50) + ' test', pageNumber: 1 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('test');
|
||||
|
||||
const match = results.find(r => r.file.path === 'overlapping.pdf');
|
||||
expect(match).toBeDefined();
|
||||
|
||||
// Should merge into one snippet since they're close together
|
||||
expect(match!.snippets.length).toBe(1);
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should not merge matches across different pages', async () => {
|
||||
const pdfFile = createMockPDFFile('cross-page.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'End of page 1 with match', pageNumber: 1 },
|
||||
{ text: 'Start of page 2 with match', pageNumber: 2 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('match');
|
||||
|
||||
const match = results.find(r => r.file.path === 'cross-page.pdf');
|
||||
expect(match).toBeDefined();
|
||||
|
||||
// Should NOT merge - different pages
|
||||
expect(match!.snippets.length).toBe(2);
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
expect(match!.snippets[1].pageNumber).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PDF error handling', () => {
|
||||
it('should handle password-protected PDFs gracefully', async () => {
|
||||
const pdfFile = createMockPDFFile('protected.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
// Mock readPDF to return error message for password-protected PDF
|
||||
const errorPages: IPageText[] = [
|
||||
{ text: 'PDF is password protected!', pageNumber: 1 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(errorPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('password');
|
||||
|
||||
const match = results.find(r => r.file.path === 'protected.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets[0].text).toContain('password protected');
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle corrupted PDF files gracefully', async () => {
|
||||
const pdfFile = createMockPDFFile('corrupted.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const errorPages: IPageText[] = [
|
||||
{ text: 'Failed to read PDF: Invalid PDF structure', pageNumber: 1 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(errorPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('PDF');
|
||||
|
||||
const match = results.find(r => r.file.path === 'corrupted.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets[0].text).toContain('Failed to read PDF');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Mixed PDF and markdown search', () => {
|
||||
it('should handle both PDF and markdown files in the same search', async () => {
|
||||
const pdfFile = createMockPDFFile('document.pdf');
|
||||
const mdFile = createMockFile('note.md');
|
||||
const folder = createMockFolder('/', [pdfFile, mdFile]);
|
||||
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'PDF page 1 with search term', pageNumber: 1 },
|
||||
{ text: 'PDF page 2 with search term', pageNumber: 2 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
mockVault.cachedRead.mockResolvedValue('Markdown content with search term');
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('search');
|
||||
|
||||
expect(results.length).toBe(2);
|
||||
|
||||
const pdfMatch = results.find(r => r.file.path === 'document.pdf');
|
||||
const mdMatch = results.find(r => r.file.path === 'note.md');
|
||||
|
||||
expect(pdfMatch).toBeDefined();
|
||||
expect(mdMatch).toBeDefined();
|
||||
|
||||
// PDF should have 2 snippets (one per page)
|
||||
expect(pdfMatch!.snippets.length).toBe(2);
|
||||
expect(pdfMatch!.snippets[0].pageNumber).toBe(1);
|
||||
expect(pdfMatch!.snippets[1].pageNumber).toBe(2);
|
||||
|
||||
// Markdown should have 1 snippet on page 1
|
||||
expect(mdMatch!.snippets.length).toBeGreaterThan(0);
|
||||
expect(mdMatch!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should preserve page numbers when sampling results', async () => {
|
||||
// Create multiple PDF files with matches
|
||||
const pdfFiles = Array.from({ length: 5 }, (_, i) =>
|
||||
createMockPDFFile(`document${i}.pdf`)
|
||||
);
|
||||
const folder = createMockFolder('/', pdfFiles);
|
||||
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'Page 1 with match', pageNumber: 1 },
|
||||
{ text: 'Page 5 with match', pageNumber: 5 },
|
||||
{ text: 'Page 10 with match', pageNumber: 10 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('match');
|
||||
|
||||
// All results should have valid page numbers
|
||||
results.forEach(result => {
|
||||
result.snippets.forEach(snippet => {
|
||||
expect(snippet.pageNumber).toBeGreaterThan(0);
|
||||
expect([1, 5, 10]).toContain(snippet.pageNumber);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('PDF snippet extraction', () => {
|
||||
it('should extract snippets with correct context from PDF pages', async () => {
|
||||
const pdfFile = createMockPDFFile('context.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const longText = 'a'.repeat(100) + 'MATCH' + 'b'.repeat(100);
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: longText, pageNumber: 5 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('MATCH');
|
||||
|
||||
const match = results.find(r => r.file.path === 'context.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets[0].text.length).toBeGreaterThan('MATCH'.length);
|
||||
expect(match!.snippets[0].text).toContain('MATCH');
|
||||
expect(match!.snippets[0].pageNumber).toBe(5);
|
||||
});
|
||||
|
||||
it('should respect snippetSizeLimit for PDF content', async () => {
|
||||
settingsService.settings.snippetSizeLimit = 20;
|
||||
|
||||
const pdfFile = createMockPDFFile('limited.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const longText = 'x'.repeat(200) + 'TARGET' + 'y'.repeat(200);
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: longText, pageNumber: 1 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('TARGET');
|
||||
|
||||
const match = results.find(r => r.file.path === 'limited.pdf');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets[0].text.length).toBeLessThanOrEqual(
|
||||
settingsService.settings.snippetSizeLimit + 10
|
||||
);
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PDF filename matching', () => {
|
||||
it('should match PDF filenames even without content matches', async () => {
|
||||
const pdfFile = createMockPDFFile('important-report.pdf');
|
||||
const folder = createMockFolder('/', [pdfFile]);
|
||||
|
||||
const mockPages: IPageText[] = [
|
||||
{ text: 'No matches in content', pageNumber: 1 }
|
||||
];
|
||||
|
||||
mockVault.getAbstractFileByPath.mockReturnValue(folder);
|
||||
mockVault.readBinary.mockResolvedValue(new ArrayBuffer(100));
|
||||
readPDFSpy.mockResolvedValue(mockPages);
|
||||
|
||||
const results = await vaultService.searchVaultFiles('important');
|
||||
|
||||
const match = results.find(r => r.file.path === 'important-report.pdf');
|
||||
expect(match).toBeDefined();
|
||||
// Should match filename even though content has no matches
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -931,6 +931,7 @@ describe('VaultService - Integration Tests', () => {
|
|||
const match = results.find(r => r.file.path === 'note1.md');
|
||||
expect(match).toBeDefined();
|
||||
expect(match!.snippets.length).toBeGreaterThan(0);
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should find filename matches', async () => {
|
||||
|
|
@ -974,6 +975,7 @@ describe('VaultService - Integration Tests', () => {
|
|||
const match = results[0];
|
||||
expect(match.snippets[0].text.length).toBeGreaterThan('MATCH'.length);
|
||||
expect(match.snippets[0].text).toContain('MATCH');
|
||||
expect(match.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should merge overlapping snippets', async () => {
|
||||
|
|
@ -1049,6 +1051,7 @@ describe('VaultService - Integration Tests', () => {
|
|||
// Snippet should be approximately snippetSizeLimit characters (10 before + 5 for MATCH + 10 after)
|
||||
// Allow some margin for the match itself
|
||||
expect(match.snippets[0].text.length).toBeLessThanOrEqual(settingsService.settings.snippetSizeLimit + 10);
|
||||
expect(match.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should perform case-insensitive search', async () => {
|
||||
|
|
@ -1080,6 +1083,7 @@ describe('VaultService - Integration Tests', () => {
|
|||
expect(match).toBeDefined();
|
||||
expect(match!.snippets.length).toBeGreaterThan(0);
|
||||
expect(match!.snippets[0].text).toContain('MAUI');
|
||||
expect(match!.snippets[0].pageNumber).toBe(1);
|
||||
});
|
||||
|
||||
it('should parse regex with word boundary patterns', async () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue