Improves suggestion generation performance

README.md

@@ -61,14 +61,13 @@ graph TD
     START((Start))
     Q_ACT_EDITOR["1. <b>Cache key</b> stems from active editor? <br>Configurable, see setting <i>Make suggestions to items in the same file</i>"]
     Q_EXCT_MATCH["2. Exact match (case sensitive) between <b>word</b> and <b>cache key</b>?"]
-    Q_WORD_IGNOR["3. <b>Word</b> is on ignore list? (case sensitive) <br>Configurable, see setting <i>Ignored words</i>"]
-    Q_WORD_SHORT["4. <b>Word</b> is too short? (Currently fixed to 3 chars)"]
-    Q_CKEY_SHORT["5. <b>Cache key</b> is too short? <br>Configurable, see setting <i>Minimum word length of suggestions</i>"]
-    Q_IS_SUBSTRG["6. <b>Word</b> is a substring of <b>cache key</b> or vice versa?"]
-    Q_WORD_UCASE["7. <b>Word</b> starts with an uppercase letter? <br>Configurable, see setting <i>Ignore occurrences which start with a lowercase letter</i>"]
-    Q_CKEY_UCASE["8. <b>Cache key</b> starts with an uppercase letter? <br>Configurable, see setting <i>Ignore suggestions which start with a lowercase letter</i>"]
-    Q_MATCH_INSV["9. Exact match (case insensitive) between <b>word</b> and <b>cache key</b>?"]
-    Q_LEN_SIMILR["10. Similarity of less than 20% length-wise between <b>word</b> and <b>cache key</b>?"]
+    Q_WORD_SHORT["3. <b>Word</b> is too short? (Currently fixed to 3 chars)"]
+    Q_CKEY_SHORT["4. <b>Cache key</b> is too short? <br>Configurable, see setting <i>Minimum word length of suggestions</i>"]
+    Q_IS_SUBSTRG["5. <b>Word</b> is a substring of <b>cache key</b> or vice versa?"]
+    Q_WORD_UCASE["6. <b>Word</b> starts with an uppercase letter? <br>Configurable, see setting <i>Ignore occurrences which start with a lowercase letter</i>"]
+    Q_CKEY_UCASE["7. <b>Cache key</b> starts with an uppercase letter? <br>Configurable, see setting <i>Ignore suggestions which start with a lowercase letter</i>"]
+    Q_MATCH_INSV["8. Exact match (case insensitive) between <b>word</b> and <b>cache key</b>?"]
+    Q_LEN_SIMILR["9. Similarity of less than 20% length-wise between <b>word</b> and <b>cache key</b>?"]
 
     STOP((STOP))
 
@@ -84,10 +83,7 @@ graph TD
     Q_ACT_EDITOR -- No --> Q_EXCT_MATCH
 
     Q_EXCT_MATCH -- Yes --> SUCCESS_1 --> STOP
-    Q_EXCT_MATCH -- No --> Q_WORD_IGNOR
-
-    Q_WORD_IGNOR -- Yes --> STOP
-    Q_WORD_IGNOR -- No --> Q_WORD_SHORT
+    Q_EXCT_MATCH -- No --> Q_WORD_SHORT
 
     Q_WORD_SHORT -- Yes --> STOP
     Q_WORD_SHORT -- No --> Q_CKEY_SHORT
@@ -110,7 +106,22 @@ graph TD
     Q_LEN_SIMILR -- No --> SUCCESS_3 --> STOP
 ```
 
-Keep in mind that these steps are processed in order. For example, take a look at the length filter in step 10. At this point, the **word** and **cache key** are already a substring of each other (step 6), meaning that this step adds things like "donut" and "donut hole punching machine manual". Not things that are in general vastly different to each other, which would create a lot of false positives.
+Then, suggestions which match the ignored words are removed:
+
+```mermaid
+graph TD
+    START((START))
+    FE["For each result"]
+    Q_WORD_IGNOR["Remove if <b>Word</b> is on ignore list (case sensitive) <br>Configurable, see setting <i>Ignored words</i>"]
+    STOP((STOP))
+
+    START --> FE
+    FE --> Q_WORD_IGNOR
+    Q_WORD_IGNOR --> FE
+    Q_WORD_IGNOR --> STOP
+```
+
+Keep in mind that these steps are processed in order. For example, take a look at the length filter in step 9. At this point, the **word** and **cache key** are already a substring of each other (step 5), meaning that this step adds things like "donut" and "donut hole punching machine manual". Not things that are in general vastly different to each other, which would create a lot of false positives.
 
 ## How to install manually
 

src/services/suggestionsService.ts

@@ -26,82 +26,99 @@ export class CrossbowSuggestionsService {
     if (!wordLookup) return [];
 
     const result: Suggestion[] = [];
-    const fileCacheEntryLookupMap = this.indexingService.getCache();
+    const cache = this.indexingService.getCache();
+    const settings = this.settingsService.getSettings();
+
+    const wordEntries = Object.entries(wordLookup);
+
+    for (let i = 0; i < wordEntries.length; i++) {
+      const [word, editorPositions] = wordEntries[i];
+      const lowercaseWord = word.toLowerCase();
 
-    Object.entries(wordLookup).forEach((entry) => {
-      const [word, editorPositions] = entry;
       const matchSet: Set<CacheMatch> = new Set();
 
-      // Find matches
-      Object.values(fileCacheEntryLookupMap).forEach((cache) => {
-        Object.keys(cache).forEach((cacheKey) => {
-          const lowercaseWord = word.toLowerCase();
+      // Find matches in the cache
+      const cacheValues = Object.values(cache);
+
+      for (let j = 0; j < cacheValues.length; j++) {
+        const cacheLookup = cacheValues[j];
+        const cacheEntries = Object.entries(cacheLookup);
+
+        for (let k = 0; k < cacheEntries.length; k++) {
+          const [cacheKey, cacheValue] = cacheEntries[k];
           const lowercaseCacheKey = cacheKey.toLowerCase();
 
+          if (matchSet.size >= 100) continue;
+
           // If reference is in the same file, and we don't want to suggest references in the same file, skip
-          if (!this.settingsService.getSettings().suggestInSameFile && cache[cacheKey].file === currentFile) return;
+          if (!this.settingsService.getSettings().suggestInSameFile && cacheValue.file === currentFile) continue;
 
           // If we have a case-sensitive exact match, we always add it, even if it does not satisfy the other filters. Say we have a chapter with a heading 'C' (eg. the programming language)
           // We want to match a word 'C' in the current editor, even if it is too short or is on the ignore list.
           if (cacheKey === word) {
-            matchSet.add({ ...cache[cacheKey], rank: '🏆' });
-            return;
+            matchSet.add({ ...cacheValue, rank: '🏆' });
+            continue;
           }
 
-          // If the word is on the ignore list, skip
-          if (this.settingsService.getSettings().ignoredWordsCaseSensisitve.includes(word)) return;
-
           // If the word is too short, skip
-          if (word.length <= 3) return;
+          if (word.length <= 3) continue;
 
           // If the cache key is too short, skip
-          if (cacheKey.length <= this.settingsService.getSettings().minimumSuggestionWordLength) return;
+          if (cacheKey.length <= settings.minimumSuggestionWordLength) continue;
 
           // If the word is not a substring of the key or the key is not a substring of the word, skip
           if ((lowercaseCacheKey.includes(lowercaseWord) || lowercaseWord.includes(lowercaseCacheKey)) === false)
-            return;
+            continue;
 
           // If the word does not start with an uppercase letter, skip
-          if (
-            this.settingsService.getSettings().ignoreOccurrencesWhichStartWithLowercaseLetter &&
-            cacheKey[0] === lowercaseCacheKey[0]
-          )
-            return;
+          if (settings.ignoreOccurrencesWhichStartWithLowercaseLetter && cacheKey[0] === lowercaseCacheKey[0]) continue;
 
           // If the cache key does not start with an uppercase letter, skip
-          if (
-            this.settingsService.getSettings().ignoreSuggestionsWhichStartWithLowercaseLetter &&
-            word[0] === lowercaseWord[0]
-          )
-            return;
+          if (settings.ignoreSuggestionsWhichStartWithLowercaseLetter && word[0] === lowercaseWord[0]) continue;
 
           // If the word is a case-insensitive exact match, add as a very good suggestion
           if (lowercaseCacheKey === lowercaseWord) {
-            matchSet.add({ ...cache[cacheKey], rank: '🥇' });
-            return;
+            matchSet.add({ ...cacheValue, rank: '🥇' });
+            continue;
           }
 
           // If the lengths differ too much, add as not-very-good suggestion
           if ((1 / cacheKey.length) * word.length <= 0.2) {
-            matchSet.add({ ...cache[cacheKey], rank: '🥉' });
-            return;
+            matchSet.add({ ...cacheValue, rank: '🥉' });
+            continue;
           }
 
           // Else, add as a mediocre suggestion
-          matchSet.add({ ...cache[cacheKey], rank: '🥈' });
-        });
-      });
+          matchSet.add({ ...cacheValue, rank: '🥈' });
+        }
+      }
 
       if (matchSet.size > 0) {
         const matches = Array.from(matchSet).map((m) => new Match(m));
         const occurrences = editorPositions.map((p) => new Occurrence(p, matches));
         result.push(new Suggestion(word, occurrences));
       }
-    });
+    }
 
     // Sort the result
     result.sort((a, b) => a.hash.localeCompare(b.hash)).forEach((suggestion) => suggestion.sortChildren());
 
+    // Remove ignored words from the result
+    return this.removeIgnoredWords(result);
+  }
+
+  private removeIgnoredWords(suggestions: Suggestion[]): Suggestion[] {
+    const ignoredWordsCaseSensisitve = this.settingsService.getSettings().ignoredWordsCaseSensisitve;
+    const ignoredWordsCache: { [key: string]: boolean } = {};
+
+    const result = suggestions.filter((suggestion) => {
+      if (ignoredWordsCache[suggestion.word] === undefined) {
+        ignoredWordsCache[suggestion.word] = !ignoredWordsCaseSensisitve.includes(suggestion.word);
+      }
+
+      return ignoredWordsCache[suggestion.word];
+    });
+
     return result;
   }
 }