milvus-io /
milvus
| @@ 49-63 (lines=15) @@ | ||
| 46 | return seq |
|
| 47 | ||
| 48 | ||
| 49 | def run_parallel(cmds, **kwargs): |
|
| 50 | """ |
|
| 51 | Run each of cmds (with shared **kwargs) using subprocess.Popen |
|
| 52 | then wait for all of them to complete. |
|
| 53 | Runs batches of multiprocessing.cpu_count() * 2 from cmds |
|
| 54 | returns a list of tuples containing each process' |
|
| 55 | returncode, stdout, stderr |
|
| 56 | """ |
|
| 57 | complete = [] |
|
| 58 | for cmds_batch in chunk(cmds, mp.cpu_count() * 2): |
|
| 59 | procs_batch = [Popen(cmd, **kwargs) for cmd in cmds_batch] |
|
| 60 | for proc in procs_batch: |
|
| 61 | stdout, stderr = proc.communicate() |
|
| 62 | complete.append((proc.returncode, stdout, stderr)) |
|
| 63 | return complete |
|
| 64 | ||
| 65 | ||
| 66 | _source_extensions = ''' |
|
| @@ 49-63 (lines=15) @@ | ||
| 46 | return seq |
|
| 47 | ||
| 48 | ||
| 49 | def run_parallel(cmds, **kwargs): |
|
| 50 | """ |
|
| 51 | Run each of cmds (with shared **kwargs) using subprocess.Popen |
|
| 52 | then wait for all of them to complete. |
|
| 53 | Runs batches of multiprocessing.cpu_count() * 2 from cmds |
|
| 54 | returns a list of tuples containing each process' |
|
| 55 | returncode, stdout, stderr |
|
| 56 | """ |
|
| 57 | complete = [] |
|
| 58 | for cmds_batch in chunk(cmds, mp.cpu_count() * 2): |
|
| 59 | procs_batch = [Popen(cmd, **kwargs) for cmd in cmds_batch] |
|
| 60 | for proc in procs_batch: |
|
| 61 | stdout, stderr = proc.communicate() |
|
| 62 | complete.append((proc.returncode, stdout, stderr)) |
|
| 63 | return complete |
|
| 64 | ||
| 65 | ||
| 66 | _source_extensions = ''' |
|
