Implement QA Inclusions filter (#777)

src/VectorStoreManager.ts

@@ -3,7 +3,7 @@ import EncryptionService from "@/encryptionService";
 import { CustomError } from "@/error";
 import EmbeddingsManager from "@/LLMProviders/embeddingManager";
 import { CopilotSettings } from "@/settings/SettingsPage";
-import { areEmbeddingModelsSame, getNotesFromTags, isPathInList } from "@/utils";
+import { areEmbeddingModelsSame, getFilePathsFromPatterns } from "@/utils";
 import VectorDBManager from "@/vectorDBManager";
 import { Embeddings } from "@langchain/core/embeddings";
 import { create, load, Orama, remove, removeMultiple, save, search } from "@orama/orama";
@@ -214,9 +214,14 @@ class VectorStoreManager {
   private updateIndexingNoticeMessage() {
     if (this.indexNoticeMessage) {
       const status = this.isIndexingPaused ? " (Paused)" : "";
-      this.indexNoticeMessage.textContent = `Copilot is indexing your vault...\n${this.indexedCount}/${this.totalFilesToIndex} files processed.${status}\nExclusions: ${
-        this.settings.qaExclusions ? this.settings.qaExclusions : "None"
-      }`;
+      const filterType = this.settings.qaInclusions
+        ? `Inclusions: ${this.settings.qaInclusions}`
+        : `Exclusions: ${this.settings.qaExclusions || "None"}`;
+
+      this.indexNoticeMessage.textContent =
+        `Copilot is indexing your vault...\n` +
+        `${this.indexedCount}/${this.totalFilesToIndex} files processed.${status}\n` +
+        filterType;
     }
   }
 
@@ -247,49 +252,36 @@ class VectorStoreManager {
     return new Notice(frag, 0);
   }
 
-  private async getExcludedFiles(): Promise<Set<string>> {
-    const excludedFiles = new Set<string>();
+  private async getFilePathsForQA(filterType: "exclusions" | "inclusions"): Promise<Set<string>> {
+    const targetFiles = new Set<string>();
 
-    if (this.settings.qaExclusions) {
+    if (filterType === "exclusions" && this.settings.qaExclusions) {
       const exclusions = this.settings.qaExclusions.split(",").map((item) => item.trim());
-
-      for (const exclusion of exclusions) {
-        if (exclusion.startsWith("#")) {
-          // Tag-based exclusion
-          const tagName = exclusion.slice(1);
-          const taggedFiles = await getNotesFromTags(this.app.vault, [tagName]);
-          taggedFiles.forEach((file) => excludedFiles.add(file.path));
-        } else if (exclusion.startsWith("*")) {
-          // File-extension-based exclusion
-          const extensionName = exclusion.slice(1);
-          this.app.vault.getFiles().forEach((file) => {
-            if (file.name.toLowerCase().contains(extensionName.toLowerCase())) {
-              excludedFiles.add(file.path);
-            }
-          });
-        } else {
-          // Path-based exclusion
-          this.app.vault.getFiles().forEach((file) => {
-            if (isPathInList(file.path, exclusion)) {
-              excludedFiles.add(file.path);
-            }
-          });
-        }
-      }
+      const excludedFilePaths = await getFilePathsFromPatterns(exclusions, this.app.vault);
+      excludedFilePaths.forEach((filePath) => targetFiles.add(filePath));
+    } else if (filterType === "inclusions" && this.settings.qaInclusions) {
+      const inclusions = this.settings.qaInclusions.split(",").map((item) => item.trim());
+      const includedFilePaths = await getFilePathsFromPatterns(inclusions, this.app.vault);
+      includedFilePaths.forEach((filePath) => targetFiles.add(filePath));
     }
 
-    return excludedFiles;
+    return targetFiles;
   }
 
-  public async getAllNonExcludedNotesContent(): Promise<string> {
+  public async getAllQAMarkdownContent(): Promise<string> {
     let allContent = "";
 
-    const excludedFiles = await this.getExcludedFiles();
-    const nonExcludedFiles = this.app.vault
-      .getMarkdownFiles()
-      .filter((file) => !excludedFiles.has(file.path));
+    const includedFiles = await this.getFilePathsForQA("inclusions");
+    const excludedFiles = await this.getFilePathsForQA("exclusions");
 
-    await Promise.all(nonExcludedFiles.map((file) => this.app.vault.cachedRead(file))).then(
+    const filteredFiles = this.app.vault.getMarkdownFiles().filter((file) => {
+      if (includedFiles.size > 0) {
+        return includedFiles.has(file.path);
+      }
+      return !excludedFiles.has(file.path);
+    });
+
+    await Promise.all(filteredFiles.map((file) => this.app.vault.cachedRead(file))).then(
       (contents) => contents.map((c) => (allContent += c + " "))
     );
 
@@ -345,7 +337,8 @@ class VectorStoreManager {
       this.isIndexingPaused = false;
       this.isIndexingCancelled = false;
 
-      const excludedFiles = await this.getExcludedFiles();
+      const includedFiles = await this.getFilePathsForQA("inclusions");
+      const excludedFiles = await this.getFilePathsForQA("exclusions");
 
       const files = this.app.vault
         .getMarkdownFiles()
@@ -353,7 +346,14 @@ class VectorStoreManager {
           if (!latestMtime || overwrite) return true;
           return file.stat.mtime > latestMtime;
         })
-        .filter((file) => !excludedFiles.has(file.path));
+        .filter((file) => {
+          if (includedFiles.size > 0) {
+            // If inclusions are specified, only include files in the inclusion set
+            return includedFiles.has(file.path);
+          }
+          // Otherwise, use exclusion filter
+          return !excludedFiles.has(file.path);
+        });
 
       const fileContents: string[] = await Promise.all(
         files.map((file) => this.app.vault.cachedRead(file))

src/constants.ts

@@ -227,6 +227,7 @@ export const DEFAULT_SETTINGS: CopilotSettings = {
   customPromptsFolder: "copilot-custom-prompts",
   indexVaultToVectorStore: VAULT_VECTOR_STORE_STRATEGY.ON_MODE_SWITCH,
   qaExclusions: "",
+  qaInclusions: "",
   chatNoteContextPath: "",
   chatNoteContextTags: [],
   debug: false,

src/main.ts

@@ -549,7 +549,7 @@ export default class CopilotPlugin extends Plugin {
 
   async countTotalTokens(): Promise<number> {
     try {
-      const allContent = await this.vectorStoreManager.getAllNonExcludedNotesContent();
+      const allContent = await this.vectorStoreManager.getAllQAMarkdownContent();
       const totalTokens = await this.chainManager.chatModelManager.countTokens(allContent);
       return totalTokens;
     } catch (error) {

src/promptUsageStrategy.ts

@@ -35,7 +35,7 @@ export class TimestampUsageStrategy implements PromptUsageStrategy {
 
   removeUnusedPrompts(existingPromptTitles: Array<string>): PromptUsageStrategy {
     for (const key in this.usageData) {
-      if (!existingPromptTitles.contains(key)) {
+      if (!existingPromptTitles.includes(key)) {
         delete this.usageData[key];
       }
     }

src/settings/SettingsPage.tsx

@@ -43,6 +43,7 @@ export interface CopilotSettings {
   enableEncryption: boolean;
   maxSourceChunks: number;
   qaExclusions: string;
+  qaInclusions: string;
   groqApiKey: string;
   enabledCommands: Record<string, { enabled: boolean; name: string }>;
   activeModels: Array<CustomModel>;

src/settings/components/QASettings.tsx

@@ -131,12 +131,19 @@ const QASettings: React.FC<QASettingsProps> = ({
         onChange={(value) => updateSettings({ embeddingRequestsPerSecond: value })}
       />
       <TextAreaComponent
-        name="Indexing Exclusions"
+        name="Exclusions"
         description="Comma separated list of paths, tags, note titles or file extension, e.g. folder1, folder1/folder2, #tag1, #tag2, [[note1]], [[note2]], *.jpg, *.excallidraw.md etc, to be excluded from the indexing process. NOTE: Tags must be in the note properties, not the note content. Files which were previously indexed will remain in the index unless you force re-index."
         placeholder="folder1, folder1/folder2, #tag1, #tag2, [[note1]], [[note2]], *.jpg, *.excallidraw.md"
         value={settings.qaExclusions}
         onChange={(value) => updateSettings({ qaExclusions: value })}
       />
+      <TextAreaComponent
+        name="Inclusions"
+        description="When specified, ONLY these paths, tags, or note titles will be indexed (comma separated). Takes precedence over exclusions. Files which were previously indexed will remain in the index unless you force re-index. Format: folder1, folder1/folder2, #tag1, #tag2, [[note1]], [[note2]]"
+        placeholder="folder1, #tag1, [[note1]]"
+        value={settings.qaInclusions}
+        onChange={(value) => updateSettings({ qaInclusions: value })}
+      />
     </div>
   );
 };

src/utils.ts

@@ -499,3 +499,35 @@ export function extractUniqueTitlesFromDocs(docs: Document[]): string[] {
 
   return Array.from(titlesSet);
 }
+
+export async function getFilePathsFromPatterns(
+  patterns: string[],
+  vault: Vault
+): Promise<string[]> {
+  const filePaths = new Set<string>();
+
+  for (const pattern of patterns) {
+    if (pattern.startsWith("#")) {
+      // Tag-based pattern
+      const taggedFiles = await getNotesFromTags(vault, [pattern]);
+      taggedFiles.forEach((file) => filePaths.add(file.path));
+    } else if (pattern.startsWith("*")) {
+      // File-extension-based pattern
+      const extensionName = pattern.slice(1);
+      vault.getFiles().forEach((file) => {
+        if (file.name.toLowerCase().endsWith(extensionName.toLowerCase())) {
+          filePaths.add(file.path);
+        }
+      });
+    } else {
+      // Path-based pattern
+      vault.getFiles().forEach((file) => {
+        if (isPathInList(file.path, pattern)) {
+          filePaths.add(file.path);
+        }
+      });
+    }
+  }
+
+  return Array.from(filePaths);
+}

tests/customPromptProcessor.test.ts

@@ -21,6 +21,16 @@ jest.mock("@/utils", () => ({
   processVariableNameForNotePath: jest.fn(),
 }));
 
+const mockUsageStrategy = {
+  incrementPromptUsage: jest.fn(),
+  getPromptUsage: jest.fn(),
+  recordUsage: jest.fn(),
+  updateUsage: jest.fn(),
+  removeUnusedPrompts: jest.fn(),
+  compare: jest.fn(),
+  save: jest.fn(),
+};
+
 describe("CustomPromptProcessor", () => {
   let processor: CustomPromptProcessor;
   let mockVault: Vault;
@@ -40,7 +50,7 @@ describe("CustomPromptProcessor", () => {
     } as TFile;
 
     // Create an instance of CustomPromptProcessor with mocked dependencies
-    processor = CustomPromptProcessor.getInstance(mockVault, mockSettings);
+    processor = CustomPromptProcessor.getInstance(mockVault, mockSettings, mockUsageStrategy);
   });
 
   it("should add 1 context and selectedText", async () => {

tests/utils.test.ts

@@ -2,6 +2,7 @@ import * as Obsidian from "obsidian";
 import { TFile } from "obsidian";
 import {
   extractNoteTitles,
+  getFilePathsFromPatterns,
   getNotesFromPath,
   getNotesFromTags,
   isFolderMatch,
@@ -312,3 +313,110 @@ describe("extractNoteTitles", () => {
     expect(result).toEqual(expected);
   });
 });
+
+describe("getFilePathsFromPatterns", () => {
+  it("should return files matching tag patterns", async () => {
+    const mockVault = new Obsidian.Vault();
+    const patterns = ["#tag1"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual(["test/test2/note1.md", "note4.md"]);
+  });
+
+  it("should return files matching file extension patterns", async () => {
+    const mockVault = new Obsidian.Vault();
+    mockVault.getFiles = jest.fn().mockReturnValue([
+      { path: "test/image1.jpg", name: "image1.jpg" },
+      { path: "test/doc.md", name: "doc.md" },
+      { path: "image2.jpg", name: "image2.jpg" },
+      { path: "test/image3.JPG", name: "image3.JPG" },
+    ]);
+
+    const patterns = ["*.jpg"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual(["test/image1.jpg", "image2.jpg", "test/image3.JPG"]);
+  });
+
+  it("should return files matching path patterns", async () => {
+    const mockVault = new Obsidian.Vault();
+    mockVault.getFiles = jest.fn().mockReturnValue([
+      { path: "test/test2/note1.md", name: "note1.md" },
+      { path: "test/note2.md", name: "note2.md" },
+      { path: "test2/note3.md", name: "note3.md" },
+    ]);
+
+    const patterns = ["test/test2"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual(["test/test2/note1.md"]);
+  });
+
+  it("should handle multiple patterns of different types", async () => {
+    const mockVault = new Obsidian.Vault();
+    mockVault.getFiles = jest.fn().mockReturnValue([
+      { path: "test/test2/note1.md", name: "note1.md" },
+      { path: "test/image1.jpg", name: "image1.jpg" },
+      { path: "note4.md", name: "note4.md" },
+      { path: "image2.jpg", name: "image2.jpg" },
+    ]);
+
+    const patterns = ["#tag1", "*.jpg", "test/test2"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual([
+      "test/test2/note1.md", // From tag1 and test/test2
+      "note4.md", // From tag1
+      "test/image1.jpg", // From *.jpg
+      "image2.jpg", // From *.jpg
+    ]);
+  });
+
+  it("should handle note title patterns with [[]]", async () => {
+    const mockVault = new Obsidian.Vault();
+    mockVault.getFiles = jest.fn().mockReturnValue([
+      { path: "test/test2/note1.md", name: "note1.md" },
+      { path: "test/note2.md", name: "note2.md" },
+    ]);
+
+    const patterns = ["[[note1]]"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual(["test/test2/note1.md"]);
+  });
+
+  it("should return empty array for non-matching patterns", async () => {
+    const mockVault = new Obsidian.Vault();
+    mockVault.getFiles = jest.fn().mockReturnValue([
+      { path: "test/test2/note1.md", name: "note1.md" },
+      { path: "test/note2.md", name: "note2.md" },
+    ]);
+
+    const patterns = ["#nonexistenttag", "*.xyz", "nonexistentfolder"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual([]);
+  });
+
+  it("should handle empty pattern array", async () => {
+    const mockVault = new Obsidian.Vault();
+    const patterns: string[] = [];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual([]);
+  });
+
+  it("should be case insensitive for file extensions", async () => {
+    const mockVault = new Obsidian.Vault();
+    mockVault.getFiles = jest.fn().mockReturnValue([
+      { path: "test/image1.JPG", name: "image1.JPG" },
+      { path: "test/image2.jpg", name: "image2.jpg" },
+      { path: "image3.Jpg", name: "image3.Jpg" },
+    ]);
+
+    const patterns = ["*.jpg"];
+
+    const result = await getFilePathsFromPatterns(patterns, mockVault);
+    expect(result).toEqual(["test/image1.JPG", "test/image2.jpg", "image3.Jpg"]);
+  });
+});