View GitHub Repository goatools

Schedule Inspection
Subscribe
Completed
Pull Request — master (#83)
by
unknown
01:34
created

p_map()   A

↳ Parent: Project

Complexity

Conditions 4

Size

Total Lines 15

Duplication

Lines 0
Ratio 0 %

Importance

Changes 1
Bugs 1 Features 0
Metric Value
cc 4
c 1
b 1
f 0
dl 0
loc 15
rs 9.2
1 
# encoding: utf-8
2 
from __future__ import print_function, division, absolute_import
3 

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    4
                

                                    
                                                    
                
                
                
import dill

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    5
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    6
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    7
                

                                    
                                                    
                
                
                
def execute(payload):

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    8
                

                                    
                                                    
                
                
                
    function, args = dill.loads(payload)

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    9
                

                                    
                                                    
                
                
                
    return dill.dumps(function(args))

            


            
                                                                                                            
                            
            
                                    
            
            

                
                    10
                

                                    
                                                    
                
                
                

            


            
                                                                                                            
                                                                
            
                                    
            
            

                
                    11
                

                                    
                                                    
                
                
                

            


            
                                                        
            
                                    
            
            

                
                    12
                

                                    
                                                    
                
                
                
def p_map(pool, function, data):

            


            
                                                        
            
                                    
            
            

                
                    13
                

                                    
                                                    
                
                
                
    """replacement for Pool.map from multiprocessing.

            


            
                                                        
            
                                    
            
            

                
                    14
                

                                    
                                                    
                
                
                
    The ability to pickle objects using the Python pickle / cPickle module

            


            
                                                        
            
                                    
            
            

                
                    15
                

                                    
                                                    
                
                
                
    limits the functions and data you can process with Pool.map.

            


            
                                                        
            
                                    
            
            

                
                    16
                

                                    
                                                    
                
                
                

            


            
                                                        
            
                                    
            
            

                
                    17
                

                                    
                                                    
                
                
                
    Here we use dill instead, which eg can pickle instance methods.

            


            
                                                        
            
                                    
            
            

                
                    18
                

                                    
                                                    
                
                
                

            


            
                                                        
            
                                    
            
            

                
                    19
                

                                    
                                                    
                
                
                
    Inspired by

            


            
                                                        
            
                                    
            
            

                
                    20
                

                                    
                                                    
                
                
                
    https://stackoverflow.com/questions/8804830/python-multiprocessing-pickling-error/24673524#24673524

            


            
                                                        
            
                                    
            
            

                
                    21
                

                                    
                                                    
                
                
                
    """

            


            
                                                        
            
                                    
            
            

                
                    22
                

                                    
                                                    
                
                
                

            


            
                                                        
            
                                    
            
            

                
                    23
                

                                    
                                                    
                
                
                
    assert isinstance(data, (list, tuple, set))

            


            
                                                        
            
                                    
            
            

                
                    24
                

                                    
                                                    
                
                
                
    payload = [dill.dumps((function, args)) for args in data]

            


            
                                                        
            
                                    
            
            

                
                    25
                

                                    
                                                    
                
                
                
    results = pool.map(execute, payload)

            


            
                                                        
            
                                    
            
            

                
                    26
                

                                    
                                                    
                
                
                
    return [dill.loads(result) for result in results]

            


            
                                                        
            
                                    
            
            

                
                    27