annif.analyzer.spacy

Complexity

Total Complexity

6

Size/Duplication

Total Lines	39
Duplicated Lines	74.36 %

Importance

Changes

0

3 Methods

Rating	Name	Duplication	Size	Complexity
A	SpacyAnalyzer.__init__()	8	8	2
A	SpacyAnalyzer.tokenize_words()	9	9	2
A	SpacyAnalyzer.normalize_word()	7	7	2

"""Simple analyzer for Annif. Only folds words to lower case."""

import spacy
from . import analyzer
import annif.util

_KEY_LOWERCASE = 'lowercase'


class SpacyAnalyzer(analyzer.Analyzer):

This code seems to be duplicated in your project.

    name = "spacy"

    def __init__(self, param, **kwargs):
        self.param = param
        self.nlp = spacy.load(param, exclude=['ner', 'parser'])
        if _KEY_LOWERCASE in kwargs:
            self.lowercase = annif.util.boolean(kwargs[_KEY_LOWERCASE])
        else:
            self.lowercase = False
        super().__init__(**kwargs)

    def tokenize_words(self, text, filter=True):
        lemmas = [lemma
                  for lemma in (token.lemma_
                                for token in self.nlp(text.strip()))
                  if (not filter or self.is_valid_token(lemma))]
        if self.lowercase:
            return [lemma.lower() for lemma in lemmas]
        else:
            return lemmas

    def normalize_word(self, word):
        doc = self.nlp(word)
        lemma = doc[:].lemma_
        if self.lowercase:
            return lemma.lower()
        else:
            return lemma