From f0c16d59056df95096d5f062805244145658e682 Mon Sep 17 00:00:00 2001 From: Simon Cambier Date: Sat, 21 Jun 2025 13:29:37 +0200 Subject: [PATCH] Always remove diacritics --- src/components/ModalInFile.svelte | 33 ++++++++++++++----------------- src/components/ModalVault.svelte | 4 ++-- src/search/search-engine.ts | 17 +++------------- src/settings/index.ts | 2 -- src/settings/settings-danger.ts | 27 ------------------------- src/settings/utils.ts | 3 --- src/tools/api.ts | 4 ++-- src/tools/text-processing.ts | 4 +--- src/tools/utils.ts | 2 +- 9 files changed, 24 insertions(+), 72 deletions(-) diff --git a/src/components/ModalInFile.svelte b/src/components/ModalInFile.svelte index 72e0f1b..bf865ab 100644 --- a/src/components/ModalInFile.svelte +++ b/src/components/ModalInFile.svelte @@ -11,10 +11,7 @@ import { onDestroy, onMount, tick } from 'svelte' import { MarkdownView, Platform } from 'obsidian' import ModalContainer from './ModalContainer.svelte' - import { - LocatorInFileModal, - LocatorVaultModal, - } from '../components/modals' + import { LocatorInFileModal, LocatorVaultModal } from '../components/modals' import ResultItemInFile from './ResultItemInFile.svelte' import { Query } from '../search/query' import { openNote } from '../tools/notes' @@ -51,8 +48,8 @@ $: (async () => { if (searchQuery) { query = new Query(searchQuery, { - ignoreDiacritics: plugin.settings.ignoreDiacritics, - ignoreArabicDiacritics: plugin.settings.ignoreArabicDiacritics, + ignoreDiacritics: true, + ignoreArabicDiacritics: true, }) note = ( @@ -163,13 +160,13 @@ (searchQuery = e.detail)} placeholder="Locator - File" - initialValue="{previousQuery}"> + initialValue={previousQuery}>
{#if Platform.isMobile} - + {/if}
@@ -179,15 +176,15 @@ {#each groupedOffsets as offset, i} + }} /> {/each} {:else}
diff --git a/src/components/ModalVault.svelte b/src/components/ModalVault.svelte index c73460b..81cb9ba 100644 --- a/src/components/ModalVault.svelte +++ b/src/components/ModalVault.svelte @@ -149,8 +149,8 @@ cancelableQuery = null } query = new Query(searchQuery, { - ignoreDiacritics: plugin.settings.ignoreDiacritics, - ignoreArabicDiacritics: plugin.settings.ignoreArabicDiacritics, + ignoreDiacritics: true, + ignoreArabicDiacritics: true, }) cancelableQuery = cancelable( new Promise(resolve => { diff --git a/src/search/search-engine.ts b/src/search/search-engine.ts index 20fcdc1..fa8aa1c 100644 --- a/src/search/search-engine.ts +++ b/src/search/search-engine.ts @@ -331,9 +331,7 @@ export class SearchEngine { results.map(async result => { const doc = await this.plugin.documentsRepository.getDocument(result.id) if (!doc) { - console.warn( - `Locator - Note "${result.id}" not in the live cache` - ) + console.warn(`Locator - Note "${result.id}" not in the live cache`) countError(true) } return doc @@ -349,12 +347,7 @@ export class SearchEngine { const title = document?.path.toLowerCase() ?? '' const content = (document?.cleanedContent ?? '').toLowerCase() return exactTerms.every( - q => - content.includes(q) || - removeDiacritics( - title, - this.plugin.settings.ignoreArabicDiacritics - ).includes(q) + q => content.includes(q) || removeDiacritics(title).includes(q) ) }) } @@ -524,11 +517,7 @@ export class SearchEngine { } return (doc as any)[fieldName] }, - processTerm: (term: string) => - (this.plugin.settings.ignoreDiacritics - ? removeDiacritics(term, this.plugin.settings.ignoreArabicDiacritics) - : term - ).toLowerCase(), + processTerm: (term: string) => removeDiacritics(term).toLowerCase(), idField: 'path', fields: [ 'basename', diff --git a/src/settings/index.ts b/src/settings/index.ts index 3edfa81..4ab302b 100644 --- a/src/settings/index.ts +++ b/src/settings/index.ts @@ -100,8 +100,6 @@ export function getDefaultSettings(app: App): LocatorSettings { hideExcluded: false, recencyBoost: RecencyCutoff.Disabled, downrankedFoldersFilters: [] as string[], - ignoreDiacritics: true, - ignoreArabicDiacritics: false, indexedFileTypes: [] as string[], displayTitle: '', PDFIndexing: false, diff --git a/src/settings/settings-danger.ts b/src/settings/settings-danger.ts index 712aeae..2ef11e4 100644 --- a/src/settings/settings-danger.ts +++ b/src/settings/settings-danger.ts @@ -15,33 +15,6 @@ export function injectSettingsDanger( new Setting(containerEl).setName('Danger Zone').setHeading() - // Ignore diacritics - new Setting(containerEl) - .setName('Ignore diacritics') - .setDesc( - htmlDescription(`Normalize diacritics in search terms. Words like "brûlée" or "žluťoučký" will be indexed as "brulee" and "zlutoucky".
- ⚠️ You probably should NOT disable this.
- ⚠️ Changing this setting will clear the cache.
- ${needsARestart}`) - ) - .addToggle(toggle => - toggle.setValue(settings.ignoreDiacritics).onChange(async v => { - await database.clearCache() - settings.ignoreDiacritics = v - await saveSettings(plugin) - }) - ) - - new Setting(containerEl) - .setName('Ignore Arabic diacritics (beta)') - .addToggle(toggle => - toggle.setValue(settings.ignoreArabicDiacritics).onChange(async v => { - await database.clearCache() - settings.ignoreArabicDiacritics = v - await saveSettings(plugin) - }) - ) - // Disable Locator const disableDesc = new DocumentFragment() disableDesc.createSpan({}, span => { diff --git a/src/settings/utils.ts b/src/settings/utils.ts index 0188e5f..4b5cf9a 100644 --- a/src/settings/utils.ts +++ b/src/settings/utils.ts @@ -41,9 +41,6 @@ export interface LocatorSettings extends WeightingSettings { recencyBoost: RecencyCutoff /** downrank files in the given folders */ downrankedFoldersFilters: string[] - /** Ignore diacritics when indexing files */ - ignoreDiacritics: boolean - ignoreArabicDiacritics: boolean /** Extensions of plain text files to index, in addition to .md */ indexedFileTypes: string[] diff --git a/src/tools/api.ts b/src/tools/api.ts index 6eb3559..58ebdc5 100644 --- a/src/tools/api.ts +++ b/src/tools/api.ts @@ -87,8 +87,8 @@ export function getApi(plugin: LocatorPlugin) { return { async search(q: string): Promise { const query = new Query(q, { - ignoreDiacritics: plugin.settings.ignoreDiacritics, - ignoreArabicDiacritics: plugin.settings.ignoreArabicDiacritics, + ignoreDiacritics: true, + ignoreArabicDiacritics: true, }) const raw = await plugin.searchEngine.getSuggestions(query) return mapResults(plugin, raw) diff --git a/src/tools/text-processing.ts b/src/tools/text-processing.ts index 35ee12e..9b2e4bd 100644 --- a/src/tools/text-processing.ts +++ b/src/tools/text-processing.ts @@ -68,9 +68,7 @@ export class TextProcessor { const reg = this.stringsToRegex(words) const originalText = text // text = text.toLowerCase().replace(new RegExp(SEPARATORS, 'gu'), ' ') - if (this.plugin.settings.ignoreDiacritics) { - text = removeDiacritics(text, this.plugin.settings.ignoreArabicDiacritics) - } + text = removeDiacritics(text) const startTime = new Date().getTime() let match: RegExpExecArray | null = null let matches: SearchMatch[] = [] diff --git a/src/tools/utils.ts b/src/tools/utils.ts index 185825f..218c156 100644 --- a/src/tools/utils.ts +++ b/src/tools/utils.ts @@ -115,7 +115,7 @@ const diacriticsRegex = new RegExp(`(?!${regexpExclude})\\p{Diacritic}`, 'gu') /** * https://stackoverflow.com/a/37511463 */ -export function removeDiacritics(str: string, arabic = false): string { +export function removeDiacritics(str: string, arabic = true): string { if (str === null || str === undefined) { return '' }