import type { QueryCombination } from 'minisearch'
import { removeStopwords } from 'stopword'
import { BRACKETS_AND_SPACE, chsRegex, SPACE_OR_PUNCTUATION } from '../globals'
import type LocatorPlugin from '../main'
import { getStopWords, splitCamelCase, splitHyphens } from '../tools/utils'

const markdownLinkExtractor = require('markdown-link-extractor')

export class Tokenizer {
  constructor(private plugin: LocatorPlugin) {}

  /**
   * Tokenization for indexing will possibly return more tokens than the original text.
   * This is because we combine different methods of tokenization to get the best results.
   * @param text
   * @returns
   */
  public tokenizeForIndexing(text: string): string[] {
    try {
      let words = this.tokenizeIntoWords(text)
      words = removeStopwords(words, getStopWords())
      let tokens = this.tokenizeIntoTokens(text, { skipChs: true })

      tokens = [
        ...tokens.flatMap(token => [
          token,
          ...splitHyphens(token),
          ...splitCamelCase(token),
        ]),
        ...words,
      ]

      return tokens
    } catch (e) {
      console.error('Error tokenizing text, skipping document', e)
      return []
    }
  }

  /**
   * Search tokenization will use the same tokenization methods as indexing,
   * but will combine each group with "OR" operators
   * @param text
   * @returns
   */
  public tokenizeForSearch(text: string): QueryCombination {
    // Extract urls and remove them from the query
    const urls: string[] = markdownLinkExtractor(text)
    text = urls.reduce((acc, url) => acc.replace(url, ''), text)

    const tokens = [...this.tokenizeIntoTokens(text), ...urls].filter(Boolean)

    return {
      combineWith: 'OR',
      queries: [
        { combineWith: 'AND', queries: tokens },
        {
          combineWith: 'AND',
          queries: this.tokenizeIntoWords(text).filter(Boolean),
        },
        { combineWith: 'AND', queries: tokens.flatMap(splitHyphens) },
        { combineWith: 'AND', queries: tokens.flatMap(splitCamelCase) },
      ],
    }
  }

  private tokenizeIntoWords(text: string, { skipChs = false } = {}): string[] {
    const tokens = text.split(BRACKETS_AND_SPACE)
    if (skipChs) return tokens
    return this.tokenizeChsWord(tokens)
  }

  private tokenizeIntoTokens(text: string, { skipChs = false } = {}): string[] {
    const tokens = text.split(SPACE_OR_PUNCTUATION)
    if (skipChs) return tokens
    return this.tokenizeChsWord(tokens)
  }

  private tokenizeChsWord(tokens: string[]): string[] {
    const segmenter = this.plugin.getChsSegmenter()
    if (!segmenter) return tokens
    return tokens.flatMap(word =>
      chsRegex.test(word) ? segmenter.cut(word, { search: true }) : [word]
    )
  }
}
