annif.analyzer.estnltk

Complexity

Total Complexity

2

Size/Duplication

Total Lines	24
Duplicated Lines	0 %

Importance

Changes

0

2 Methods

Rating	Name	Duplication	Size	Complexity
A	EstNLTKAnalyzer.__init__()	0	3	1
A	EstNLTKAnalyzer.tokenize_words()	0	9	1

"""EstNLTK analyzer for Annif which uses EstNLTK for lemmatization"""

from __future__ import annotations

from . import analyzer


class EstNLTKAnalyzer(analyzer.Analyzer):
    name = "estnltk"

    def __init__(self, param: str, **kwargs) -> None:
        self.param = param
        super().__init__(**kwargs)

    def tokenize_words(self, text: str, filter: bool = True) -> list[str]:
        import estnltk

        txt = estnltk.Text(text.strip())
        txt.tag_layer()
        return [
            lemma
            for lemma in [lemmas[0] for lemmas in txt.lemma]
            if (not filter or self.is_valid_token(lemma))
        ]