From 48d014b5cc88cf130db2e0356a0751e1ff39d016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Sch=C3=B6dler?= Date: Sun, 21 May 2023 00:12:36 +0200 Subject: [PATCH] Improves suggestion generation performance --- README.md | 37 ++++++++----- src/services/suggestionsService.ts | 85 ++++++++++++++++++------------ 2 files changed, 75 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 4168b3e..faefe08 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,13 @@ graph TD START((Start)) Q_ACT_EDITOR["1. Cache key stems from active editor?
Configurable, see setting Make suggestions to items in the same file"] Q_EXCT_MATCH["2. Exact match (case sensitive) between word and cache key?"] - Q_WORD_IGNOR["3. Word is on ignore list? (case sensitive)
Configurable, see setting Ignored words"] - Q_WORD_SHORT["4. Word is too short? (Currently fixed to 3 chars)"] - Q_CKEY_SHORT["5. Cache key is too short?
Configurable, see setting Minimum word length of suggestions"] - Q_IS_SUBSTRG["6. Word is a substring of cache key or vice versa?"] - Q_WORD_UCASE["7. Word starts with an uppercase letter?
Configurable, see setting Ignore occurrences which start with a lowercase letter"] - Q_CKEY_UCASE["8. Cache key starts with an uppercase letter?
Configurable, see setting Ignore suggestions which start with a lowercase letter"] - Q_MATCH_INSV["9. Exact match (case insensitive) between word and cache key?"] - Q_LEN_SIMILR["10. Similarity of less than 20% length-wise between word and cache key?"] + Q_WORD_SHORT["3. Word is too short? (Currently fixed to 3 chars)"] + Q_CKEY_SHORT["4. Cache key is too short?
Configurable, see setting Minimum word length of suggestions"] + Q_IS_SUBSTRG["5. Word is a substring of cache key or vice versa?"] + Q_WORD_UCASE["6. Word starts with an uppercase letter?
Configurable, see setting Ignore occurrences which start with a lowercase letter"] + Q_CKEY_UCASE["7. Cache key starts with an uppercase letter?
Configurable, see setting Ignore suggestions which start with a lowercase letter"] + Q_MATCH_INSV["8. Exact match (case insensitive) between word and cache key?"] + Q_LEN_SIMILR["9. Similarity of less than 20% length-wise between word and cache key?"] STOP((STOP)) @@ -84,10 +83,7 @@ graph TD Q_ACT_EDITOR -- No --> Q_EXCT_MATCH Q_EXCT_MATCH -- Yes --> SUCCESS_1 --> STOP - Q_EXCT_MATCH -- No --> Q_WORD_IGNOR - - Q_WORD_IGNOR -- Yes --> STOP - Q_WORD_IGNOR -- No --> Q_WORD_SHORT + Q_EXCT_MATCH -- No --> Q_WORD_SHORT Q_WORD_SHORT -- Yes --> STOP Q_WORD_SHORT -- No --> Q_CKEY_SHORT @@ -110,7 +106,22 @@ graph TD Q_LEN_SIMILR -- No --> SUCCESS_3 --> STOP ``` -Keep in mind that these steps are processed in order. For example, take a look at the length filter in step 10. At this point, the **word** and **cache key** are already a substring of each other (step 6), meaning that this step adds things like "donut" and "donut hole punching machine manual". Not things that are in general vastly different to each other, which would create a lot of false positives. +Then, suggestions which match the ignored words are removed: + +```mermaid +graph TD + START((START)) + FE["For each result"] + Q_WORD_IGNOR["Remove if Word is on ignore list (case sensitive)
Configurable, see setting Ignored words"] + STOP((STOP)) + + START --> FE + FE --> Q_WORD_IGNOR + Q_WORD_IGNOR --> FE + Q_WORD_IGNOR --> STOP +``` + +Keep in mind that these steps are processed in order. For example, take a look at the length filter in step 9. At this point, the **word** and **cache key** are already a substring of each other (step 5), meaning that this step adds things like "donut" and "donut hole punching machine manual". Not things that are in general vastly different to each other, which would create a lot of false positives. ## How to install manually diff --git a/src/services/suggestionsService.ts b/src/services/suggestionsService.ts index c7aaf47..506bbf7 100644 --- a/src/services/suggestionsService.ts +++ b/src/services/suggestionsService.ts @@ -26,82 +26,99 @@ export class CrossbowSuggestionsService { if (!wordLookup) return []; const result: Suggestion[] = []; - const fileCacheEntryLookupMap = this.indexingService.getCache(); + const cache = this.indexingService.getCache(); + const settings = this.settingsService.getSettings(); + + const wordEntries = Object.entries(wordLookup); + + for (let i = 0; i < wordEntries.length; i++) { + const [word, editorPositions] = wordEntries[i]; + const lowercaseWord = word.toLowerCase(); - Object.entries(wordLookup).forEach((entry) => { - const [word, editorPositions] = entry; const matchSet: Set = new Set(); - // Find matches - Object.values(fileCacheEntryLookupMap).forEach((cache) => { - Object.keys(cache).forEach((cacheKey) => { - const lowercaseWord = word.toLowerCase(); + // Find matches in the cache + const cacheValues = Object.values(cache); + + for (let j = 0; j < cacheValues.length; j++) { + const cacheLookup = cacheValues[j]; + const cacheEntries = Object.entries(cacheLookup); + + for (let k = 0; k < cacheEntries.length; k++) { + const [cacheKey, cacheValue] = cacheEntries[k]; const lowercaseCacheKey = cacheKey.toLowerCase(); + if (matchSet.size >= 100) continue; + // If reference is in the same file, and we don't want to suggest references in the same file, skip - if (!this.settingsService.getSettings().suggestInSameFile && cache[cacheKey].file === currentFile) return; + if (!this.settingsService.getSettings().suggestInSameFile && cacheValue.file === currentFile) continue; // If we have a case-sensitive exact match, we always add it, even if it does not satisfy the other filters. Say we have a chapter with a heading 'C' (eg. the programming language) // We want to match a word 'C' in the current editor, even if it is too short or is on the ignore list. if (cacheKey === word) { - matchSet.add({ ...cache[cacheKey], rank: '🏆' }); - return; + matchSet.add({ ...cacheValue, rank: '🏆' }); + continue; } - // If the word is on the ignore list, skip - if (this.settingsService.getSettings().ignoredWordsCaseSensisitve.includes(word)) return; - // If the word is too short, skip - if (word.length <= 3) return; + if (word.length <= 3) continue; // If the cache key is too short, skip - if (cacheKey.length <= this.settingsService.getSettings().minimumSuggestionWordLength) return; + if (cacheKey.length <= settings.minimumSuggestionWordLength) continue; // If the word is not a substring of the key or the key is not a substring of the word, skip if ((lowercaseCacheKey.includes(lowercaseWord) || lowercaseWord.includes(lowercaseCacheKey)) === false) - return; + continue; // If the word does not start with an uppercase letter, skip - if ( - this.settingsService.getSettings().ignoreOccurrencesWhichStartWithLowercaseLetter && - cacheKey[0] === lowercaseCacheKey[0] - ) - return; + if (settings.ignoreOccurrencesWhichStartWithLowercaseLetter && cacheKey[0] === lowercaseCacheKey[0]) continue; // If the cache key does not start with an uppercase letter, skip - if ( - this.settingsService.getSettings().ignoreSuggestionsWhichStartWithLowercaseLetter && - word[0] === lowercaseWord[0] - ) - return; + if (settings.ignoreSuggestionsWhichStartWithLowercaseLetter && word[0] === lowercaseWord[0]) continue; // If the word is a case-insensitive exact match, add as a very good suggestion if (lowercaseCacheKey === lowercaseWord) { - matchSet.add({ ...cache[cacheKey], rank: '🥇' }); - return; + matchSet.add({ ...cacheValue, rank: '🥇' }); + continue; } // If the lengths differ too much, add as not-very-good suggestion if ((1 / cacheKey.length) * word.length <= 0.2) { - matchSet.add({ ...cache[cacheKey], rank: '🥉' }); - return; + matchSet.add({ ...cacheValue, rank: '🥉' }); + continue; } // Else, add as a mediocre suggestion - matchSet.add({ ...cache[cacheKey], rank: '🥈' }); - }); - }); + matchSet.add({ ...cacheValue, rank: '🥈' }); + } + } if (matchSet.size > 0) { const matches = Array.from(matchSet).map((m) => new Match(m)); const occurrences = editorPositions.map((p) => new Occurrence(p, matches)); result.push(new Suggestion(word, occurrences)); } - }); + } // Sort the result result.sort((a, b) => a.hash.localeCompare(b.hash)).forEach((suggestion) => suggestion.sortChildren()); + // Remove ignored words from the result + return this.removeIgnoredWords(result); + } + + private removeIgnoredWords(suggestions: Suggestion[]): Suggestion[] { + const ignoredWordsCaseSensisitve = this.settingsService.getSettings().ignoredWordsCaseSensisitve; + const ignoredWordsCache: { [key: string]: boolean } = {}; + + const result = suggestions.filter((suggestion) => { + if (ignoredWordsCache[suggestion.word] === undefined) { + ignoredWordsCache[suggestion.word] = !ignoredWordsCaseSensisitve.includes(suggestion.word); + } + + return ignoredWordsCache[suggestion.word]; + }); + return result; } }