annif.parallel.get_pool() - Code Metrics - Inspection of "Implement hyperparameter optimization of ensemble..." - NatLibFi/Annif - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#414)

by Osma

created 2020-07-21 12:37 UTC

annif.parallel.get_pool() A

↳ Parent: annif.parallel

Complexity

Conditions

Size

Total Lines	14
Code Lines	8

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	8
dl	0
loc	14
rs	10
c	0
b	0
f	0
cc	3
nop	1

"""Parallel processing functionality for Annif"""


import multiprocessing
import multiprocessing.dummy


class ProjectSuggestMap:
    """A utility class that can be used to wrap one or more projects and
    provide a mapping method that converts Document objects to suggestions.
    Intended to be used with the multiprocessing module."""

    def __init__(
            self,
            registry,
            project_ids,
            backend_params,
            limit,
            threshold):
        self.registry = registry
        self.project_ids = project_ids
        self.backend_params = backend_params
        self.limit = limit
        self.threshold = threshold

    def suggest(self, doc):
        filtered_hits = {}
        for project_id in self.project_ids:
            project = self.registry.get_project(project_id)
            hits = project.suggest(doc.text, self.backend_params)
            filtered_hits[project_id] = hits.filter(
                project.subjects, self.limit, self.threshold)
        return (filtered_hits, doc.uris, doc.labels)


def get_pool(n_jobs):
    """return a suitable multiprocessing pool class, and the correct jobs
    argument for its constructor, for the given amount of parallel jobs"""

    if n_jobs < 1:
        n_jobs = None
        pool_class = multiprocessing.Pool
    elif n_jobs == 1:
        # use the dummy wrapper around threading to avoid subprocess overhead
        pool_class = multiprocessing.dummy.Pool
    else:
        pool_class = multiprocessing.Pool

    return n_jobs, pool_class


1			"""Parallel processing functionality for Annif"""
2
3
4			import multiprocessing
5			import multiprocessing.dummy
6
7
8			class ProjectSuggestMap:
9			"""A utility class that can be used to wrap one or more projects and
10			provide a mapping method that converts Document objects to suggestions.
11			Intended to be used with the multiprocessing module."""
12
13			def __init__(
14			self,
15			registry,
16			project_ids,
17			backend_params,
18			limit,
19			threshold):
20			self.registry = registry
21			self.project_ids = project_ids
22			self.backend_params = backend_params
23			self.limit = limit
24			self.threshold = threshold
25
26			def suggest(self, doc):
27			filtered_hits = {}
28			for project_id in self.project_ids:
29			project = self.registry.get_project(project_id)
30			hits = project.suggest(doc.text, self.backend_params)
31			filtered_hits[project_id] = hits.filter(
32			project.subjects, self.limit, self.threshold)
33			return (filtered_hits, doc.uris, doc.labels)
34
35
36			def get_pool(n_jobs):
37			"""return a suitable multiprocessing pool class, and the correct jobs
38			argument for its constructor, for the given amount of parallel jobs"""
39
40			if n_jobs < 1:
41			n_jobs = None
42			pool_class = multiprocessing.Pool
43			elif n_jobs == 1:
44			# use the dummy wrapper around threading to avoid subprocess overhead
45			pool_class = multiprocessing.dummy.Pool
46			else:
47			pool_class = multiprocessing.Pool
48
49			return n_jobs, pool_class
50

NatLibFi / Annif

Pull Request — master (#414)

annif.parallel.get_pool() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like