annif.analyzer.spacy

Complexity

Total Complexity

5

Size/Duplication

Total Lines	31
Duplicated Lines	0 %

Importance

Changes

0

4 Methods

Rating	Name	Duplication	Size	Complexity
A	SpacyAnalyzer.__init__()	0	6	1
A	SpacyAnalyzer.tokenize_words()	0	5	2
A	SpacyAnalyzer.normalize_word()	0	3	1
A	SpacyAnalyzer.tokenize_sentences()	0	3	1

"""Simple analyzer for Annif. Only folds words to lower case."""

import spacy
from spacy.tokens import Doc, Span
from . import analyzer


class SpacyAnalyzer(analyzer.Analyzer):
    name = "spacy"

    def __init__(self, param, **kwargs):
        self.param = param
        self.nlp = spacy.load(param, exclude=['ner', 'parser'])
        # we need a way to split sentences, now that parser is excluded
        self.nlp.add_pipe('sentencizer')
        super().__init__(**kwargs)

    def tokenize_sentences(self, text):
        doc = self.nlp(text)
        return list(doc.sents)

    def tokenize_words(self, text):
        if not isinstance(text, (Doc, Span)):
            text = self.nlp(text)
        return [lemma for lemma in (token.lemma_ for token in text)
                if self.is_valid_token(lemma)]

    def normalize_word(self, word):
        doc = self.nlp(word)
        return doc[:].lemma_