cleaning

src/components/modals.ts

@@ -164,10 +164,8 @@ export class LocatorVaultModal extends LocatorModal {
       ?.editor.getSelection()
 
     plugin.searchHistory.getHistory().then(history => {
-      // Previously searched query (if enabled in settings)
-      const previous = plugin.settings.showPreviousQueryResults
-        ? history[0]
-        : null
+      // Previously searched query
+      const previous = history[0]
 
       // Instantiate and display the Svelte component
       const cmp = mount(ModalVault, {

src/main.ts

@@ -274,6 +274,7 @@ export default class LocatorPlugin extends Plugin {
     }
 
     console.timeEnd('Indexing total time')
+    console.log(`Indexed ${diff.toAdd.length} files, removed ${diff.toRemove.length} files`)
     if (diff.toAdd.length >= 1000 && isCacheEnabled()) {
       new Notice(`Your files have been indexed.`)
     }

src/search/tokenizer.ts

@@ -16,9 +16,11 @@ export class Tokenizer {
    */
   public tokenizeForIndexing(text: string): string[] {
     try {
-      const words = this.tokenizeWords(text)
+      const lang = eld.detectLanguage(text)
+      console.log(lang)
+      const words = this.tokenizeIntoWords(text)
+      let tokens = this.tokenizeIntoTokens(text, { skipChs: true })
 
-      let tokens = this.tokenizeTokens(text, { skipChs: true })
       tokens = [
         ...tokens.flatMap(token => [
           token,
@@ -28,9 +30,6 @@ export class Tokenizer {
         ...words,
       ]
 
-      // Remove duplicates
-      tokens = [...new Set(tokens)]
-
       return tokens
     } catch (e) {
       console.error('Error tokenizing text, skipping document', e)
@@ -49,7 +48,7 @@ export class Tokenizer {
     const urls: string[] = markdownLinkExtractor(text)
     text = urls.reduce((acc, url) => acc.replace(url, ''), text)
 
-    const tokens = [...this.tokenizeTokens(text), ...urls].filter(Boolean)
+    const tokens = [...this.tokenizeIntoTokens(text), ...urls].filter(Boolean)
 
     return {
       combineWith: 'OR',
@@ -57,7 +56,7 @@ export class Tokenizer {
         { combineWith: 'AND', queries: tokens },
         {
           combineWith: 'AND',
-          queries: this.tokenizeWords(text).filter(Boolean),
+          queries: this.tokenizeIntoWords(text).filter(Boolean),
         },
         { combineWith: 'AND', queries: tokens.flatMap(splitHyphens) },
         { combineWith: 'AND', queries: tokens.flatMap(splitCamelCase) },
@@ -65,13 +64,13 @@ export class Tokenizer {
     }
   }
 
-  private tokenizeWords(text: string, { skipChs = false } = {}): string[] {
+  private tokenizeIntoWords(text: string, { skipChs = false } = {}): string[] {
     const tokens = text.split(BRACKETS_AND_SPACE)
     if (skipChs) return tokens
     return this.tokenizeChsWord(tokens)
   }
 
-  private tokenizeTokens(text: string, { skipChs = false } = {}): string[] {
+  private tokenizeIntoTokens(text: string, { skipChs = false } = {}): string[] {
     const tokens = text.split(SPACE_OR_PUNCTUATION)
     if (skipChs) return tokens
     return this.tokenizeChsWord(tokens)

src/settings/index.ts

@@ -97,7 +97,6 @@ export function getDefaultSettings(app: App): LocatorSettings {
     maxEmbeds: 5,
     renderLineReturnInExcerpts: true,
     showCreateButton: false,
-    showPreviousQueryResults: true,
     simpleSearch: false,
     fuzziness: '1',
 

src/settings/settings-behavior.ts

@@ -27,17 +27,6 @@ export function injectSettingsBehavior(
       })
     )
 
-  // Show previous query results
-  new Setting(containerEl)
-    .setName('Show previous query results')
-    .setDesc('Re-executes the previous query when opening Locator.')
-    .addToggle(toggle =>
-      toggle.setValue(settings.showPreviousQueryResults).onChange(async v => {
-        settings.showPreviousQueryResults = v
-        await saveSettings(plugin)
-      })
-    )
-
   // Respect excluded files
   new Setting(containerEl)
     .setName('Respect Obsidian\'s "Excluded Files"')

src/settings/utils.ts

@@ -68,8 +68,6 @@ export interface LocatorSettings extends WeightingSettings {
   renderLineReturnInExcerpts: boolean
   /** Enable a "create note" button in the Vault Search modal */
   showCreateButton: boolean
-  /** Re-execute the last query when opening Locator */
-  showPreviousQueryResults: boolean
   /** Key for the welcome message when Obsidian is updated. A message is only shown once. */
   welcomeMessage: string
   /** If a query returns 0 result, try again with more relax conditions */