annif.analyzer.spacy.SpacyAnalyzer.tokenize_words() - Code Metrics - Inspection of "Add spaCy analyzer" - NatLibFi/Annif - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#527)

by Osma

created 2022-01-18 14:45 UTC

SpacyAnalyzer.tokenize_words() A

↳ Parent: annif.analyzer.spacy

Complexity

Conditions

Size

Total Lines	7
Code Lines	6

Duplication

Lines	7
Ratio	100 %

Importance

Changes

Metric	Value
cc	2
eloc	6
nop	2
dl	7
loc	7
rs	10
c	0
b	0
f	0

"""Simple analyzer for Annif. Only folds words to lower case."""

import spacy
from . import analyzer
import annif.util

_KEY_LOWERCASE = 'lowercase'


class SpacyAnalyzer(analyzer.Analyzer):

    name = "spacy"

    def __init__(self, param, **kwargs):
        self.param = param
        self.nlp = spacy.load(param, exclude=['ner', 'parser'])
        if _KEY_LOWERCASE in kwargs:
            self.lowercase = annif.util.boolean(kwargs[_KEY_LOWERCASE])
        else:
            self.lowercase = False
        super().__init__(**kwargs)

    def tokenize_words(self, text):
        lemmas = [lemma for lemma in (token.lemma_ for token in self.nlp(text))
                  if self.is_valid_token(lemma)]
        if self.lowercase:
            return [lemma.lower() for lemma in lemmas]
        else:
            return lemmas

    def normalize_word(self, word):
        doc = self.nlp(word)
        lemma = doc[:].lemma_
        if self.lowercase:
            return lemma.lower()
        else:
            return lemma


1		"""Simple analyzer for Annif. Only folds words to lower case."""
2
3		import spacy
4		from . import analyzer
5		import annif.util
6
7		_KEY_LOWERCASE = 'lowercase'
8
9
10	View Code Duplication	class SpacyAnalyzer(analyzer.Analyzer):
		0 ignored issues – show Duplication introduced 2022-01-18 14:32 UTC by Report Bug Copy Issue Report This code seems to be duplicated in your project. Loading history...
11		name = "spacy"
12
13		def __init__(self, param, **kwargs):
14		self.param = param
15		self.nlp = spacy.load(param, exclude=['ner', 'parser'])
16		if _KEY_LOWERCASE in kwargs:
17		self.lowercase = annif.util.boolean(kwargs[_KEY_LOWERCASE])
18		else:
19		self.lowercase = False
20		super().__init__(**kwargs)
21
22		def tokenize_words(self, text):
23		lemmas = [lemma for lemma in (token.lemma_ for token in self.nlp(text))
24		if self.is_valid_token(lemma)]
25		if self.lowercase:
26		return [lemma.lower() for lemma in lemmas]
27		else:
28		return lemmas
29
30		def normalize_word(self, word):
31		doc = self.nlp(word)
32		lemma = doc[:].lemma_
33		if self.lowercase:
34		return lemma.lower()
35		else:
36		return lemma
37

NatLibFi / Annif

Pull Request — master (#527)

SpacyAnalyzer.tokenize_words() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like