View GitHub Repository Annif

Schedule Inspection
Subscribe
Passed
Pull Request — master (#527)
by Osma
02:34
created

SpacyAnalyzer.tokenize_words()   A

↳ Parent: annif.analyzer.spacy

Complexity

Conditions 2

Size

Total Lines 9
Code Lines 8

Duplication

Lines 9
Ratio 100 %

Importance

Changes 0
Metric Value
cc 2
eloc 8
nop 3
dl 9
loc 9
rs 10
c 0
b 0
f 0
1 
"""spaCy analyzer for Annif which uses spaCy for lemmatization"""
2 

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    3
                

                                    
                                                    
                
                
                
import spacy

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    4
                

                                    
                                                    
                
                
                
from . import analyzer

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    5
                

                                    
                                                    
                
                
                
import annif.util

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    6
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    7
                

                                    
                                                    
                
                
                
_KEY_LOWERCASE = 'lowercase'

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    8
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    9
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                                                    
                                                                                                        
            
            

                
                    10
                

                                    
                                                    
                
                View Code Duplication
                
class SpacyAnalyzer(analyzer.Analyzer):

            


                            

                    
                        
                    
                    
                    
                    
                                                                                                    
                        

                            
                            
0 ignored issues

                            –
                            show
                        


                        

                                                                                        




    

        
            Duplication
    
    
    
        introduced 
                            by 
            

        

        

            
        

        

            

                

                    
                
                
            


        

    

    

        
                        

                            
                        

                                        
This code seems to be duplicated in your project.
                                            

                                            
                    
                                                    

                     Loading history...
                

                    


    




                                                                                    


                    		
                

            
                                                                                                            
                            
            
                                    
            
            

                
                    11
                

                                    
                                                    
                
                
                
    name = "spacy"

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    12
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    13
                

                                    
                                                    
                
                
                
    def __init__(self, param, **kwargs):

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    14
                

                                    
                                                    
                
                
                
        self.param = param

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    15
                

                                    
                                                    
                
                
                
        self.nlp = spacy.load(param, exclude=['ner', 'parser'])

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    16
                

                                    
                                                    
                
                
                
        if _KEY_LOWERCASE in kwargs:

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    17
                

                                    
                                                    
                
                
                
            self.lowercase = annif.util.boolean(kwargs[_KEY_LOWERCASE])

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    18
                

                                    
                                                    
                
                
                
        else:

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    19
                

                                    
                                                    
                
                
                
            self.lowercase = False

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    20
                

                                    
                                                    
                
                
                
        super().__init__(**kwargs)

            


            
                                                                                                            
                                                                
            
                                    
            
            

                
                    21
                

                                    
                                                    
                
                
                

            


            
                                                        
            
                                    
            
            

                
                    22
                

                                    
                                                    
                
                
                
    def tokenize_words(self, text, filter=True):

            


            
                                                        
            
                                    
            
            

                
                    23
                

                                    
                                                    
                
                
                
        lemmas = [lemma

            


            
                                                        
            
                                    
            
            

                
                    24
                

                                    
                                                    
                
                
                
                  for lemma in (token.lemma_

            


            
                                                        
            
                                    
            
            

                
                    25
                

                                    
                                                    
                
                
                
                                for token in self.nlp(text.strip()))

            


            
                                                        
            
                                    
            
            

                
                    26
                

                                    
                                                    
                
                
                
                  if (not filter or self.is_valid_token(lemma))]

            


            
                                                        
            
                                    
            
            

                
                    27
                

                                    
                                                    
                
                
                
        if self.lowercase:

            


            
                                                        
            
                                    
            
            

                
                    28
                

                                    
                                                    
                
                
                
            return [lemma.lower() for lemma in lemmas]

            


            
                                                        
            
                                    
            
            

                
                    29
                

                                    
                                                    
                
                
                
        else:

            


            
                                                        
            
                                    
            
            

                
                    30
                

                                    
                                                    
                
                
                
            return lemmas

            


            
                                                        
            
                                    
            
            

                
                    31