annif.analyzer.spacy

Complexity

Total Complexity

3

Size/Duplication

Total Lines	22
Duplicated Lines	0 %

Importance

Changes

0

3 Methods

Rating	Name	Duplication	Size	Complexity
A	SpacyAnalyzer.__init__()	0	4	1
A	SpacyAnalyzer.tokenize_words()	0	3	1
A	SpacyAnalyzer.normalize_word()	0	3	1

"""Simple analyzer for Annif. Only folds words to lower case."""

import spacy
from . import analyzer


class SpacyAnalyzer(analyzer.Analyzer):
    name = "spacy"

    def __init__(self, param, **kwargs):
        self.param = param
        self.nlp = spacy.load(param, exclude=['ner', 'parser'])
        super().__init__(**kwargs)

    def tokenize_words(self, text):
        return [lemma for lemma in (token.lemma_ for token in self.nlp(text))
                if self.is_valid_token(lemma)]

    def normalize_word(self, word):
        doc = self.nlp(word)
        return doc[:].lemma_