remove stopwords

src/search/tokenizer.ts

@@ -1,7 +1,8 @@
 import type { QueryCombination } from 'minisearch'
+import { removeStopwords } from 'stopword'
 import { BRACKETS_AND_SPACE, chsRegex, SPACE_OR_PUNCTUATION } from '../globals'
 import type LocatorPlugin from '../main'
-import { splitCamelCase, splitHyphens } from '../tools/utils'
+import { getStopWords, splitCamelCase, splitHyphens } from '../tools/utils'
 
 const markdownLinkExtractor = require('markdown-link-extractor')
 
@@ -16,9 +17,8 @@ export class Tokenizer {
    */
   public tokenizeForIndexing(text: string): string[] {
     try {
-      const lang = eld.detectLanguage(text)
-      console.log(lang)
-      const words = this.tokenizeIntoWords(text)
+      let words = this.tokenizeIntoWords(text)
+      words = removeStopwords(words, getStopWords())
       let tokens = this.tokenizeIntoTokens(text, { skipChs: true })
 
       tokens = [

src/tools/utils.ts

@@ -1,3 +1,4 @@
+import { type BinaryLike, createHash } from 'crypto'
 import {
   type CachedMetadata,
   getAllTags,
@@ -5,9 +6,9 @@ import {
   parseFrontMatterAliases,
   Platform,
 } from 'obsidian'
-import { isSearchMatch, type SearchMatch } from '../globals'
-import { type BinaryLike, createHash } from 'crypto'
 import { md5 } from 'pure-md5'
+import { eng, fra } from 'stopword'
+import { isSearchMatch, type SearchMatch } from '../globals'
 
 export function pathWithoutFilename(path: string): string {
   const split = path.split('/')
@@ -279,3 +280,13 @@ export const countError = (() => {
     }
   }
 })()
+
+let stopWords: string[] = []
+export function getStopWords(): string[] {
+  if (!stopWords.length) {
+    stopWords = [...eng, ...fra]
+    // Remove duplicates
+    stopWords = [...new Set(stopWords)]
+  }
+  return stopWords
+}