annif.analyzer.spacy.SpacyAnalyzer.__init__() - Code Metrics - Inspection of "Merge pull request #527 from NatLibFi/issue374-spa..." - NatLibFi/Annif - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( cf5fd0...1b4762 )

by Osma

created 2022-01-24 08:05 UTC

annif.analyzer.spacy.SpacyAnalyzer.init() A

↳ Parent: annif.analyzer.spacy

Complexity

Conditions

Size

Total Lines	13
Code Lines	12

Duplication

Lines	13
Ratio	100 %

Importance

Changes

Metric	Value
cc	3
eloc	12
nop	3
dl	13
loc	13
rs	9.8
c	0
b	0
f	0

"""spaCy analyzer for Annif which uses spaCy for lemmatization"""

import spacy
from . import analyzer
from annif.exception import OperationFailedException
import annif.util

_KEY_LOWERCASE = 'lowercase'


class SpacyAnalyzer(analyzer.Analyzer):

    name = "spacy"

    def __init__(self, param, **kwargs):
        self.param = param
        try:
            self.nlp = spacy.load(param, exclude=['ner', 'parser'])
        except IOError as err:
            raise OperationFailedException(
                f"Loading spaCy model '{param}' failed - " +
                f"please download the model.\n{err}")
        if _KEY_LOWERCASE in kwargs:
            self.lowercase = annif.util.boolean(kwargs[_KEY_LOWERCASE])
        else:
            self.lowercase = False
        super().__init__(**kwargs)

    def tokenize_words(self, text, filter=True):
        lemmas = [lemma
                  for lemma in (token.lemma_
                                for token in self.nlp(text.strip()))
                  if (not filter or self.is_valid_token(lemma))]
        if self.lowercase:
            return [lemma.lower() for lemma in lemmas]
        else:
            return lemmas


1		"""spaCy analyzer for Annif which uses spaCy for lemmatization"""
2
3		import spacy
4		from . import analyzer
5		from annif.exception import OperationFailedException
6		import annif.util
7
8		_KEY_LOWERCASE = 'lowercase'
9
10
11	View Code Duplication	class SpacyAnalyzer(analyzer.Analyzer):
		0 ignored issues – show Duplication introduced 2022-01-18 14:32 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
12		name = "spacy"
13
14		def __init__(self, param, **kwargs):
15		self.param = param
16		try:
17		self.nlp = spacy.load(param, exclude=['ner', 'parser'])
18		except IOError as err:
19		raise OperationFailedException(
20		f"Loading spaCy model '{param}' failed - " +
21		f"please download the model.\n{err}")
22		if _KEY_LOWERCASE in kwargs:
23		self.lowercase = annif.util.boolean(kwargs[_KEY_LOWERCASE])
24		else:
25		self.lowercase = False
26		super().__init__(**kwargs)
27
28		def tokenize_words(self, text, filter=True):
29		lemmas = [lemma
30		for lemma in (token.lemma_
31		for token in self.nlp(text.strip()))
32		if (not filter or self.is_valid_token(lemma))]
33		if self.lowercase:
34		return [lemma.lower() for lemma in lemmas]
35		else:
36		return lemmas
37

NatLibFi / Annif

Push — master ( cf5fd0...1b4762 )

annif.analyzer.spacy.SpacyAnalyzer.__init__() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

annif.analyzer.spacy.SpacyAnalyzer.init() A