annif.analyzer.estnltk

Complexity

Total Complexity

2

Size/Duplication

Total Lines	29
Duplicated Lines	0 %

Importance

Changes

0

2 Methods

Rating	Name	Duplication	Size	Complexity
A	EstNLTKAnalyzer.__init__()	0	3	1
A	EstNLTKAnalyzer.tokenize_words()	0	11	1

"""EstNLTK analyzer for Annif which uses EstNLTK for lemmatization"""

from __future__ import annotations

import annif.util
from annif.exception import OperationFailedException

from . import analyzer


class EstNLTKAnalyzer(analyzer.Analyzer):
    name = "estnltk"

    def __init__(self, param: str, **kwargs) -> None:
        self.param = param
        super().__init__(**kwargs)

    def tokenize_words(self, text: str, filter: bool = True) -> list[str]:
        import estnltk

        txt = estnltk.Text(text.strip())
        txt.tag_layer()
        lemmas = [
            lemma
            for lemma in [l[0] for l in txt.lemma]
            if (not filter or self.is_valid_token(lemma))
        ]
        return lemmas