|
1
|
|
|
"""Distribution module for parallel processing in hyperparameter optimization. |
|
2
|
|
|
|
|
3
|
|
|
This module provides various methods for distributing optimization processes |
|
4
|
|
|
across multiple cores or threads, including multiprocessing, pathos, and joblib. |
|
5
|
|
|
|
|
6
|
|
|
Author: Simon Blanke |
|
7
|
|
|
Email: [email protected] |
|
8
|
|
|
License: MIT License |
|
9
|
|
|
""" |
|
10
|
|
|
|
|
11
|
|
|
from sys import platform |
|
12
|
|
|
|
|
13
|
|
|
from tqdm import tqdm |
|
14
|
|
|
|
|
15
|
|
|
if platform.startswith("linux"): |
|
16
|
|
|
initializer = tqdm.set_lock |
|
17
|
|
|
initargs = (tqdm.get_lock(),) |
|
18
|
|
|
else: |
|
19
|
|
|
initializer = None |
|
20
|
|
|
initargs = () |
|
21
|
|
|
|
|
22
|
|
|
|
|
23
|
|
|
def single_process(process_func, process_infos): |
|
24
|
|
|
"""Execute processes sequentially in a single thread.""" |
|
25
|
|
|
return [process_func(*info) for info in process_infos] |
|
26
|
|
|
|
|
27
|
|
|
|
|
28
|
|
|
def multiprocessing_wrapper(process_func, process_infos, n_processes): |
|
29
|
|
|
"""Execute processes using multiprocessing library.""" |
|
30
|
|
|
import multiprocessing as mp |
|
31
|
|
|
|
|
32
|
|
|
with mp.Pool(n_processes, initializer=initializer, initargs=initargs) as pool: |
|
33
|
|
|
return pool.map(process_func, process_infos) |
|
34
|
|
|
|
|
35
|
|
|
|
|
36
|
|
|
def pathos_wrapper(process_func, search_processes_paras, n_processes): |
|
37
|
|
|
"""Execute processes using pathos multiprocessing library.""" |
|
38
|
|
|
import pathos.multiprocessing as pmp |
|
39
|
|
|
|
|
40
|
|
|
with pmp.Pool(n_processes, initializer=initializer, initargs=initargs) as pool: |
|
41
|
|
|
return pool.map(process_func, search_processes_paras) |
|
42
|
|
|
|
|
43
|
|
|
|
|
44
|
|
|
def joblib_wrapper(process_func, search_processes_paras, n_processes): |
|
45
|
|
|
"""Execute processes using joblib parallel processing.""" |
|
46
|
|
|
from joblib import Parallel, delayed |
|
47
|
|
|
|
|
48
|
|
|
jobs = [delayed(process_func)(*info_dict) for info_dict in search_processes_paras] |
|
49
|
|
|
return Parallel(n_jobs=n_processes)(jobs) |
|
50
|
|
|
|
SimonBlanke /
Hyperactive
