coalib.processes.autoapply_actions() - Code Metrics - Inspection of "Makman2/auto-apply2" - coala-analyzer/coala - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#1170)

by Mischa

created 2015-12-30 17:09 UTC

coalib.processes.autoapply_actions() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	7
dl	0
loc	62
rs	6.2296

How to fix Long Method

import multiprocessing
import queue
import os
import platform
import subprocess

from coalib.collecting.Collectors import collect_files
from coalib.collecting import Dependencies
from coalib.misc.StringConverter import StringConverter
from coalib.output.printers import LOG_LEVEL
from coalib.processes.BearRunning import run
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.results.Result import Result
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange
from coalib.settings.Setting import path_list
from coalib.processes.LogPrinterThread import LogPrinterThread
from coalib.results.result_actions.OpenEditorAction import OpenEditorAction
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.results.result_actions.ShowPatchAction import ShowPatchAction


ACTIONS = [OpenEditorAction,
           ApplyPatchAction,
           PrintDebugMessageAction,
           ShowPatchAction]


def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    # cpu_count is not implemented for some CPU architectures/OSes
    except NotImplementedError:  # pragma: no cover
        return 2


def fill_queue(queue_fill, any_list):
    """
    Takes element from a list and populates a queue with those elements.

    :param queue_fill: The queue to be filled.
    :param any_list:   List containing the elements.
    """
    for elem in any_list:
        queue_fill.put(elem)


def get_running_processes(processes):
    return sum((1 if process.is_alive() else 0) for process in processes)


def create_process_group(command_array, **kwargs):
    if platform.system() == "Windows":  # pragma: no cover
        proc = subprocess.Popen(
            command_array,
            creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
            **kwargs)
    else:
        proc = subprocess.Popen(command_array,
                                preexec_fn=os.setsid,
                                **kwargs)
    return proc


def get_default_actions(section):
    """
    Parses the key `default_actions` in the given section.

    :param section:    The section where to parse from.
    :return:           A dict with the bearname as keys and their default
                       actions as values and another dict that contains bears
                       and invalid action names.
    """
    try:
        default_actions = dict(section["default_actions"])
    except IndexError:
        return {}, {}

    action_dict = {action.get_metadata().name: action for action in ACTIONS}
    invalid_action_set = default_actions.values() - action_dict.keys()
    invalid_actions = {}
    if len(invalid_action_set) != 0:
        invalid_actions = {
            bear: action
            for bear, action in default_actions.items()
            if action in invalid_action_set}
        for invalid in invalid_actions.keys():
            del default_actions[invalid]

    actions = {bearname: action_dict[action_name]
               for bearname, action_name in default_actions.items()}
    return actions, invalid_actions


def autoapply_actions(results,
                      file_dict,
                      file_diff_dict,
                      section,
                      log_printer):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """

    default_actions, invalid_actions = get_default_actions(section)

    for bearname, actionname in invalid_actions.items():
        log_printer.warn("Selected default action {} for bear {} does "
                         "not exist. Ignoring action.".format(
                             repr(actionname),
                             repr(bearname)))

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            action = default_actions[result.origin]
        except KeyError:
            not_processed_results.append(result)
            continue

        if not action.is_applicable(result, file_dict, file_diff_dict):
            log_printer.warn("Selected default action {} for bear {} is not "
                             "applicable. Action not applied.".format(
                                 repr(action.get_metadata().name),
                                 repr(result.origin)))
            not_processed_results.append(result)
            continue

        try:
            action().apply_from_section(result,
                                        file_dict,
                                        file_diff_dict,
                                        section)
            log_printer.info("Applied {} for {}.".format(
                repr(action.get_metadata().name),
                repr(result.origin)))
        except Exception as ex:
            log_printer.log_exception(
                "Failed to execute action {}.".format(
                    repr(action.get_metadata().name)),
                ex)
            log_printer.debug("-> for result " + repr(result) + ".")

    return not_processed_results


def print_result(results,
                 file_dict,
                 retval,
                 print_results,
                 section,
                 log_printer,
                 file_diff_dict,
                 ignore_ranges):
    """
    Takes the results produced by each bear and gives them to the print_results
    method to present to the user.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param retval:         It is True if no results were yielded ever before.
                           If it is False this function will return False no
                           matter what happens. Else it depends on if this
                           invocation yields results.
    :param print_results:  A function that prints all given results appropriate
                           to the output medium.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param ignore_ranges:  A list of SourceRanges. Results that affect code in
                           any of those ranges will be ignored.
    :return:               Returns False if any results were yielded. Else
                           True.
    """
    min_severity_str = str(section.get('min_severity', 'INFO')).upper()
    min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
    results = list(filter(lambda result:
                              type(result) is Result and
                              result.severity >= min_severity and
                              not result.to_ignore(ignore_ranges),
                          results))

    results = autoapply_actions(results,
                                file_dict,
                                file_diff_dict,
                                section,
                                log_printer)

    print_results(log_printer, section, results, file_dict, file_diff_dict)
    return retval or len(results) > 0


def get_file_dict(filename_list, log_printer):
    """
    Reads all files into a dictionary.

    :param filename_list: List of names of paths to files to get contents of.
    :param log_printer:   The logger which logs errors.
    :return:              Reads the content of each file into a dictionary
                          with filenames as keys.
    """
    file_dict = {}
    for filename in filename_list:
        try:
            with open(filename, "r", encoding="utf-8") as _file:
                file_dict[filename] = _file.readlines()
        except UnicodeDecodeError:
            log_printer.warn("Failed to read file '{}'. It seems to contain "
                             "non-unicode characters. Leaving it "
                             "out.".format(filename))
        except Exception as exception:  # pragma: no cover
            log_printer.log_exception("Failed to read file '{}' because of "
                                      "an unknown error. Leaving it "
                                      "out.".format(filename),
                                      exception,
                                      log_level=LOG_LEVEL.WARNING)

    return file_dict


def filter_raising_callables(it, exception, *args, **kwargs):
    """
    Filters all callable items inside the given iterator that raise the
    given exceptions.

    :param it:        The iterator to filter.
    :param exception: The (tuple of) exception(s) to filter for.
    :param args:      Positional arguments to pass to the callable.
    :param kwargs:    Keyword arguments to pass to the callable.
    """
    for elem in it:
        try:
            yield elem(*args, **kwargs)
        except exception:
            pass


def instantiate_bears(section,
                      local_bear_list,
                      global_bear_list,
                      file_dict,
                      message_queue):
    """
    Instantiates each bear with the arguments it needs.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bear classes to instantiate.
    :param global_bear_list: List of global bear classes to instantiate.
    :param file_dict:        Dictionary containing filenames and their
                             contents.
    :param message_queue:    Queue responsible to maintain the messages
                             delivered by the bears.
    :return:                 The local and global bear instance lists.
    """
    local_bear_list = [bear
                       for bear in filter_raising_callables(
                           local_bear_list,
                           RuntimeError,
                           section,
                           message_queue,
                           timeout=0.1)]

    global_bear_list = [bear
                        for bear in filter_raising_callables(
                            global_bear_list,
                            RuntimeError,
                            file_dict,
                            section,
                            message_queue,
                            timeout=0.1)]

    return local_bear_list, global_bear_list


def instantiate_processes(section,
                          local_bear_list,
                          global_bear_list,
                          job_count,
                          log_printer):
    """
    Instantiate the number of processes that will run bears which will be
    responsible for running bears in a multiprocessing environment.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bears belonging to the section.
    :param global_bear_list: List of global bears belonging to the section.
    :param job_count:        Max number of processes to create.
    :param log_printer:      The log printer to warn to.
    :return:                 A tuple containing a list of processes,
                             and the arguments passed to each process which are
                             the same for each object.
    """
    filename_list = collect_files(path_list(section.get('files', "")),
                                  path_list(section.get('ignore', "")))
    file_dict = get_file_dict(filename_list, log_printer)

    manager = multiprocessing.Manager()
    global_bear_queue = multiprocessing.Queue()
    filename_queue = multiprocessing.Queue()
    local_result_dict = manager.dict()
    global_result_dict = manager.dict()
    message_queue = multiprocessing.Queue()
    control_queue = multiprocessing.Queue()

    bear_runner_args = {"file_name_queue": filename_queue,
                        "local_bear_list": local_bear_list,
                        "global_bear_list": global_bear_list,
                        "global_bear_queue": global_bear_queue,
                        "file_dict": file_dict,
                        "local_result_dict": local_result_dict,
                        "global_result_dict": global_result_dict,
                        "message_queue": message_queue,
                        "control_queue": control_queue,
                        "timeout": 0.1}

    local_bear_list[:], global_bear_list[:] = instantiate_bears(
        section,
        local_bear_list,
        global_bear_list,
        file_dict,
        message_queue)

    fill_queue(filename_queue, file_dict.keys())
    fill_queue(global_bear_queue, range(len(global_bear_list)))

    return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
             for i in range(job_count)],
            bear_runner_args)


def get_ignore_scope(line, keyword):
    """
    Retrieves the bears that are to be ignored defined in the given line.
    :param line:    The line containing the ignore declaration.
    :param keyword: The keyword that was found. Everything after the rightmost
                    occurrence of it will be considered for the scope.
    :return:        A list of lower cased bearnames or an empty list (-> "all")
    """
    toignore = line[line.rfind(keyword) + len(keyword):]
    if toignore.startswith("all"):
        return []
    else:
        return list(StringConverter(toignore, list_delimiters=', '))


def yield_ignore_ranges(file_dict):
    """
    Yields tuples of affected bears and a SourceRange that shall be ignored for
    those.

    :param file_dict: The file dictionary.
    """
    for filename, file in file_dict.items():
        start = None
        bears = []
        for line_number, line in enumerate(file, start=1):
            line = line.lower()
            if "start ignoring " in line:
                start = line_number
                bears = get_ignore_scope(line, "start ignoring ")
            elif "stop ignoring" in line:
                if start:
                    yield (bears,
                           SourceRange.from_values(filename,
                                                   start,
                                                   end_line=line_number))
            elif "ignore " in line:
                yield (get_ignore_scope(line, "ignore "),
                       SourceRange.from_values(filename,
                                               line_number,
                                               end_line=line_number+1))


def process_queues(processes,
                   control_queue,
                   local_result_dict,
                   global_result_dict,
                   file_dict,
                   print_results,
                   section,
                   log_printer):
    """
    Iterate the control queue and send the results recieved to the print_result
    method so that they can be presented to the user.

    :param processes:          List of processes which can be used to run
                               Bears.
    :param control_queue:      Containing control elements that indicate
                               whether there is a result available and which
                               bear it belongs to.
    :param local_result_dict:  Dictionary containing results respective to
                               local bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param global_result_dict: Dictionary containing results respective to
                               global bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param file_dict:          Dictionary containing file contents with
                               filename as keys.
    :param print_results:      Prints all given results appropriate to the
                               output medium.
    :return:                   Return True if all bears execute succesfully and
                               Results were delivered to the user. Else False.
    """
    file_diff_dict = {}
    running_processes = get_running_processes(processes)
    retval = False
    # Number of processes working on local bears
    local_processes = len(processes)
    global_result_buffer = []
    ignore_ranges = list(yield_ignore_ranges(file_dict))

    # One process is the logger thread
    while local_processes > 1 and running_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
                local_processes -= 1
            elif control_elem == CONTROL_ELEMENT.LOCAL:
                assert local_processes != 0
                retval = print_result(local_result_dict[index],
                                      file_dict,
                                      retval,
                                      print_results,
                                      section,
                                      log_printer,
                                      file_diff_dict,
                                      ignore_ranges)
            elif control_elem == CONTROL_ELEMENT.GLOBAL:
                global_result_buffer.append(index)
        except queue.Empty:
            running_processes = get_running_processes(processes)

    # Flush global result buffer
    for elem in global_result_buffer:
        retval = print_result(global_result_dict[elem],
                              file_dict,
                              retval,
                              print_results,
                              section,
                              log_printer,
                              file_diff_dict,
                              ignore_ranges)

    running_processes = get_running_processes(processes)
    # One process is the logger thread
    while running_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.GLOBAL:
                retval = print_result(global_result_dict[index],
                                      file_dict,
                                      retval,
                                      print_results,
                                      section,
                                      log_printer,
                                      file_diff_dict,
                                      ignore_ranges)
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
                running_processes = get_running_processes(processes)

        except queue.Empty:
            running_processes = get_running_processes(processes)

    return retval


def execute_section(section,
                    global_bear_list,
                    local_bear_list,
                    print_results,
                    log_printer):
    """
    Executes the section with the given bears.

    The execute_section method does the following things:
    1. Prepare a Process
      * Load files
      * Create queues
    2. Spawn up one or more Processes
    3. Output results from the Processes
    4. Join all processes

    :param section:          The section to execute.
    :param global_bear_list: List of global bears belonging to the section.
    :param local_bear_list:  List of local bears belonging to the section.
    :param print_results:    Prints all given results appropriate to the
                             output medium.
    :param log_printer:      The log_printer to warn to.
    :return:                 Tuple containing a bool (True if results were
                             yielded, False otherwise), a Manager.dict
                             containing all local results(filenames are key)
                             and a Manager.dict containing all global bear
                             results (bear names are key) as well as the
                             file dictionary.
    """
    local_bear_list = Dependencies.resolve(local_bear_list)
    global_bear_list = Dependencies.resolve(global_bear_list)

    try:
        running_processes = int(section['jobs'])
    except ValueError:
        log_printer.warn("Unable to convert setting 'jobs' into a number. "
                         "Falling back to CPU count.")
        running_processes = get_cpu_count()
    except IndexError:
        running_processes = get_cpu_count()

    processes, arg_dict = instantiate_processes(section,
                                                local_bear_list,
                                                global_bear_list,
                                                running_processes,
                                                log_printer)

    logger_thread = LogPrinterThread(arg_dict["message_queue"],
                                     log_printer)
    # Start and join the logger thread along with the processes to run bears
    processes.append(logger_thread)

    for runner in processes:
        runner.start()

    try:
        return (process_queues(processes,
                               arg_dict["control_queue"],
                               arg_dict["local_result_dict"],
                               arg_dict["global_result_dict"],
                               arg_dict["file_dict"],
                               print_results,
                               section,
                               log_printer),
                arg_dict["local_result_dict"],
                arg_dict["global_result_dict"],
                arg_dict["file_dict"])
    finally:
        logger_thread.running = False

        for runner in processes:
            runner.join()


1			import multiprocessing
2			import queue
3			import os
4			import platform
5			import subprocess
6
7			from coalib.collecting.Collectors import collect_files
8			from coalib.collecting import Dependencies
9			from coalib.misc.StringConverter import StringConverter
10			from coalib.output.printers import LOG_LEVEL
11			from coalib.processes.BearRunning import run
12			from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
13			from coalib.results.Result import Result
14			from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
15			from coalib.results.SourceRange import SourceRange
16			from coalib.settings.Setting import path_list
17			from coalib.processes.LogPrinterThread import LogPrinterThread
18			from coalib.results.result_actions.OpenEditorAction import OpenEditorAction
19			from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
20			from coalib.results.result_actions.PrintDebugMessageAction import (
21			PrintDebugMessageAction)
22			from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
23
24
25			ACTIONS = [OpenEditorAction,
26			ApplyPatchAction,
27			PrintDebugMessageAction,
28			ShowPatchAction]
29
30
31			def get_cpu_count():
32			try:
33			return multiprocessing.cpu_count()
34			# cpu_count is not implemented for some CPU architectures/OSes
35			except NotImplementedError: # pragma: no cover
36			return 2
37
38
39			def fill_queue(queue_fill, any_list):
40			"""
41			Takes element from a list and populates a queue with those elements.
42
43			:param queue_fill: The queue to be filled.
44			:param any_list: List containing the elements.
45			"""
46			for elem in any_list:
47			queue_fill.put(elem)
48
49
50			def get_running_processes(processes):
51			return sum((1 if process.is_alive() else 0) for process in processes)
52
53
54			def create_process_group(command_array, **kwargs):
55			if platform.system() == "Windows": # pragma: no cover
56			proc = subprocess.Popen(
57			command_array,
58			creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
59			**kwargs)
60			else:
61			proc = subprocess.Popen(command_array,
62			preexec_fn=os.setsid,
63			**kwargs)
64			return proc
65
66
67			def get_default_actions(section):
68			"""
69			Parses the key `default_actions` in the given section.
70
71			:param section: The section where to parse from.
72			:return: A dict with the bearname as keys and their default
73			actions as values and another dict that contains bears
74			and invalid action names.
75			"""
76			try:
77			default_actions = dict(section["default_actions"])
78			except IndexError:
79			return {}, {}
80
81			action_dict = {action.get_metadata().name: action for action in ACTIONS}
82			invalid_action_set = default_actions.values() - action_dict.keys()
83			invalid_actions = {}
84			if len(invalid_action_set) != 0:
85			invalid_actions = {
86			bear: action
87			for bear, action in default_actions.items()
88			if action in invalid_action_set}
89			for invalid in invalid_actions.keys():
90			del default_actions[invalid]
91
92			actions = {bearname: action_dict[action_name]
93			for bearname, action_name in default_actions.items()}
94			return actions, invalid_actions
95
96
97			def autoapply_actions(results,
98			file_dict,
99			file_diff_dict,
100			section,
101			log_printer):
102			"""
103			Auto-applies actions like defined in the given section.
104
105			:param results: A list of results.
106			:param file_dict: A dictionary containing the name of files and its
107			contents.
108			:param file_diff_dict: A dictionary that contains filenames as keys and
109			diff objects as values.
110			:param section: The section.
111			:param log_printer: A log printer instance to log messages on.
112			:return: A list of unprocessed results.
113			"""
114
115			default_actions, invalid_actions = get_default_actions(section)
116
117			for bearname, actionname in invalid_actions.items():
118			log_printer.warn("Selected default action {} for bear {} does "
119			"not exist. Ignoring action.".format(
120			repr(actionname),
121			repr(bearname)))
122
123			if len(default_actions) == 0:
124			# There's nothing to auto-apply.
125			return results
126
127			not_processed_results = []
128			for result in results:
129			try:
130			action = default_actions[result.origin]
131			except KeyError:
132			not_processed_results.append(result)
133			continue
134
135			if not action.is_applicable(result, file_dict, file_diff_dict):
136			log_printer.warn("Selected default action {} for bear {} is not "
137			"applicable. Action not applied.".format(
138			repr(action.get_metadata().name),
139			repr(result.origin)))
140			not_processed_results.append(result)
141			continue
142
143			try:
144			action().apply_from_section(result,
145			file_dict,
146			file_diff_dict,
147			section)
148			log_printer.info("Applied {} for {}.".format(
149			repr(action.get_metadata().name),
150			repr(result.origin)))
151			except Exception as ex:
152			log_printer.log_exception(
153			"Failed to execute action {}.".format(
154			repr(action.get_metadata().name)),
155			ex)
156			log_printer.debug("-> for result " + repr(result) + ".")
157
158			return not_processed_results
159
160
161			def print_result(results,
162			file_dict,
163			retval,
164			print_results,
165			section,
166			log_printer,
167			file_diff_dict,
168			ignore_ranges):
169			"""
170			Takes the results produced by each bear and gives them to the print_results
171			method to present to the user.
172
173			:param results: A list of results.
174			:param file_dict: A dictionary containing the name of files and its
175			contents.
176			:param retval: It is True if no results were yielded ever before.
177			If it is False this function will return False no
178			matter what happens. Else it depends on if this
179			invocation yields results.
180			:param print_results: A function that prints all given results appropriate
181			to the output medium.
182			:param file_diff_dict: A dictionary that contains filenames as keys and
183			diff objects as values.
184			:param ignore_ranges: A list of SourceRanges. Results that affect code in
185			any of those ranges will be ignored.
186			:return: Returns False if any results were yielded. Else
187			True.
188			"""
189			min_severity_str = str(section.get('min_severity', 'INFO')).upper()
190			min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
191			results = list(filter(lambda result:
192			type(result) is Result and
193			result.severity >= min_severity and
194			not result.to_ignore(ignore_ranges),
195			results))
196
197			results = autoapply_actions(results,
198			file_dict,
199			file_diff_dict,
200			section,
201			log_printer)
202
203			print_results(log_printer, section, results, file_dict, file_diff_dict)
204			return retval or len(results) > 0
205
206
207			def get_file_dict(filename_list, log_printer):
208			"""
209			Reads all files into a dictionary.
210
211			:param filename_list: List of names of paths to files to get contents of.
212			:param log_printer: The logger which logs errors.
213			:return: Reads the content of each file into a dictionary
214			with filenames as keys.
215			"""
216			file_dict = {}
217			for filename in filename_list:
218			try:
219			with open(filename, "r", encoding="utf-8") as _file:
220			file_dict[filename] = _file.readlines()
221			except UnicodeDecodeError:
222			log_printer.warn("Failed to read file '{}'. It seems to contain "
223			"non-unicode characters. Leaving it "
224			"out.".format(filename))
225			except Exception as exception: # pragma: no cover
226			log_printer.log_exception("Failed to read file '{}' because of "
227			"an unknown error. Leaving it "
228			"out.".format(filename),
229			exception,
230			log_level=LOG_LEVEL.WARNING)
231
232			return file_dict
233
234
235			def filter_raising_callables(it, exception, args, *kwargs):
236			"""
237			Filters all callable items inside the given iterator that raise the
238			given exceptions.
239
240			:param it: The iterator to filter.
241			:param exception: The (tuple of) exception(s) to filter for.
242			:param args: Positional arguments to pass to the callable.
243			:param kwargs: Keyword arguments to pass to the callable.
244			"""
245			for elem in it:
246			try:
247			yield elem(args, *kwargs)
248			except exception:
249			pass
250
251
252			def instantiate_bears(section,
253			local_bear_list,
254			global_bear_list,
255			file_dict,
256			message_queue):
257			"""
258			Instantiates each bear with the arguments it needs.
259
260			:param section: The section the bears belong to.
261			:param local_bear_list: List of local bear classes to instantiate.
262			:param global_bear_list: List of global bear classes to instantiate.
263			:param file_dict: Dictionary containing filenames and their
264			contents.
265			:param message_queue: Queue responsible to maintain the messages
266			delivered by the bears.
267			:return: The local and global bear instance lists.
268			"""
269			local_bear_list = [bear
270			for bear in filter_raising_callables(
271			local_bear_list,
272			RuntimeError,
273			section,
274			message_queue,
275			timeout=0.1)]
276
277			global_bear_list = [bear
278			for bear in filter_raising_callables(
279			global_bear_list,
280			RuntimeError,
281			file_dict,
282			section,
283			message_queue,
284			timeout=0.1)]
285
286			return local_bear_list, global_bear_list
287
288
289			def instantiate_processes(section,
290			local_bear_list,
291			global_bear_list,
292			job_count,
293			log_printer):
294			"""
295			Instantiate the number of processes that will run bears which will be
296			responsible for running bears in a multiprocessing environment.
297
298			:param section: The section the bears belong to.
299			:param local_bear_list: List of local bears belonging to the section.
300			:param global_bear_list: List of global bears belonging to the section.
301			:param job_count: Max number of processes to create.
302			:param log_printer: The log printer to warn to.
303			:return: A tuple containing a list of processes,
304			and the arguments passed to each process which are
305			the same for each object.
306			"""
307			filename_list = collect_files(path_list(section.get('files', "")),
308			path_list(section.get('ignore', "")))
309			file_dict = get_file_dict(filename_list, log_printer)
310
311			manager = multiprocessing.Manager()
312			global_bear_queue = multiprocessing.Queue()
313			filename_queue = multiprocessing.Queue()
314			local_result_dict = manager.dict()
315			global_result_dict = manager.dict()
316			message_queue = multiprocessing.Queue()
317			control_queue = multiprocessing.Queue()
318
319			bear_runner_args = {"file_name_queue": filename_queue,
320			"local_bear_list": local_bear_list,
321			"global_bear_list": global_bear_list,
322			"global_bear_queue": global_bear_queue,
323			"file_dict": file_dict,
324			"local_result_dict": local_result_dict,
325			"global_result_dict": global_result_dict,
326			"message_queue": message_queue,
327			"control_queue": control_queue,
328			"timeout": 0.1}
329
330			local_bear_list[:], global_bear_list[:] = instantiate_bears(
331			section,
332			local_bear_list,
333			global_bear_list,
334			file_dict,
335			message_queue)
336
337			fill_queue(filename_queue, file_dict.keys())
338			fill_queue(global_bear_queue, range(len(global_bear_list)))
339
340			return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
341			for i in range(job_count)],
342			bear_runner_args)
343
344
345			def get_ignore_scope(line, keyword):
346			"""
347			Retrieves the bears that are to be ignored defined in the given line.
348			:param line: The line containing the ignore declaration.
349			:param keyword: The keyword that was found. Everything after the rightmost
350			occurrence of it will be considered for the scope.
351			:return: A list of lower cased bearnames or an empty list (-> "all")
352			"""
353			toignore = line[line.rfind(keyword) + len(keyword):]
354			if toignore.startswith("all"):
355			return []
356			else:
357			return list(StringConverter(toignore, list_delimiters=', '))
358
359
360			def yield_ignore_ranges(file_dict):
361			"""
362			Yields tuples of affected bears and a SourceRange that shall be ignored for
363			those.
364
365			:param file_dict: The file dictionary.
366			"""
367			for filename, file in file_dict.items():
368			start = None
369			bears = []
370			for line_number, line in enumerate(file, start=1):
371			line = line.lower()
372			if "start ignoring " in line:
373			start = line_number
374			bears = get_ignore_scope(line, "start ignoring ")
375			elif "stop ignoring" in line:
376			if start:
377			yield (bears,
378			SourceRange.from_values(filename,
379			start,
380			end_line=line_number))
381			elif "ignore " in line:
382			yield (get_ignore_scope(line, "ignore "),
383			SourceRange.from_values(filename,
384			line_number,
385			end_line=line_number+1))
386
387
388			def process_queues(processes,
389			control_queue,
390			local_result_dict,
391			global_result_dict,
392			file_dict,
393			print_results,
394			section,
395			log_printer):
396			"""
397			Iterate the control queue and send the results recieved to the print_result
398			method so that they can be presented to the user.
399
400			:param processes: List of processes which can be used to run
401			Bears.
402			:param control_queue: Containing control elements that indicate
403			whether there is a result available and which
404			bear it belongs to.
405			:param local_result_dict: Dictionary containing results respective to
406			local bears. It is modified by the processes
407			i.e. results are added to it by multiple
408			processes.
409			:param global_result_dict: Dictionary containing results respective to
410			global bears. It is modified by the processes
411			i.e. results are added to it by multiple
412			processes.
413			:param file_dict: Dictionary containing file contents with
414			filename as keys.
415			:param print_results: Prints all given results appropriate to the
416			output medium.
417			:return: Return True if all bears execute succesfully and
418			Results were delivered to the user. Else False.
419			"""
420			file_diff_dict = {}
421			running_processes = get_running_processes(processes)
422			retval = False
423			# Number of processes working on local bears
424			local_processes = len(processes)
425			global_result_buffer = []
426			ignore_ranges = list(yield_ignore_ranges(file_dict))
427
428			# One process is the logger thread
429			while local_processes > 1 and running_processes > 1:
430			try:
431			control_elem, index = control_queue.get(timeout=0.1)
432
433			if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
434			local_processes -= 1
435			elif control_elem == CONTROL_ELEMENT.LOCAL:
436			assert local_processes != 0
437			retval = print_result(local_result_dict[index],
438			file_dict,
439			retval,
440			print_results,
441			section,
442			log_printer,
443			file_diff_dict,
444			ignore_ranges)
445			elif control_elem == CONTROL_ELEMENT.GLOBAL:
446			global_result_buffer.append(index)
447			except queue.Empty:
448			running_processes = get_running_processes(processes)
449
450			# Flush global result buffer
451			for elem in global_result_buffer:
452			retval = print_result(global_result_dict[elem],
453			file_dict,
454			retval,
455			print_results,
456			section,
457			log_printer,
458			file_diff_dict,
459			ignore_ranges)
460
461			running_processes = get_running_processes(processes)
462			# One process is the logger thread
463			while running_processes > 1:
464			try:
465			control_elem, index = control_queue.get(timeout=0.1)
466
467			if control_elem == CONTROL_ELEMENT.GLOBAL:
468			retval = print_result(global_result_dict[index],
469			file_dict,
470			retval,
471			print_results,
472			section,
473			log_printer,
474			file_diff_dict,
475			ignore_ranges)
476			else:
477			assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
478			running_processes = get_running_processes(processes)
479
480			except queue.Empty:
481			running_processes = get_running_processes(processes)
482
483			return retval
484
485
486			def execute_section(section,
487			global_bear_list,
488			local_bear_list,
489			print_results,
490			log_printer):
491			"""
492			Executes the section with the given bears.
493
494			The execute_section method does the following things:
495			1. Prepare a Process
496			* Load files
497			* Create queues
498			2. Spawn up one or more Processes
499			3. Output results from the Processes
500			4. Join all processes
501
502			:param section: The section to execute.
503			:param global_bear_list: List of global bears belonging to the section.
504			:param local_bear_list: List of local bears belonging to the section.
505			:param print_results: Prints all given results appropriate to the
506			output medium.
507			:param log_printer: The log_printer to warn to.
508			:return: Tuple containing a bool (True if results were
509			yielded, False otherwise), a Manager.dict
510			containing all local results(filenames are key)
511			and a Manager.dict containing all global bear
512			results (bear names are key) as well as the
513			file dictionary.
514			"""
515			local_bear_list = Dependencies.resolve(local_bear_list)
516			global_bear_list = Dependencies.resolve(global_bear_list)
517
518			try:
519			running_processes = int(section['jobs'])
520			except ValueError:
521			log_printer.warn("Unable to convert setting 'jobs' into a number. "
522			"Falling back to CPU count.")
523			running_processes = get_cpu_count()
524			except IndexError:
525			running_processes = get_cpu_count()
526
527			processes, arg_dict = instantiate_processes(section,
528			local_bear_list,
529			global_bear_list,
530			running_processes,
531			log_printer)
532
533			logger_thread = LogPrinterThread(arg_dict["message_queue"],
534			log_printer)
535			# Start and join the logger thread along with the processes to run bears
536			processes.append(logger_thread)
537
538			for runner in processes:
539			runner.start()
540
541			try:
542			return (process_queues(processes,
543			arg_dict["control_queue"],
544			arg_dict["local_result_dict"],
545			arg_dict["global_result_dict"],
546			arg_dict["file_dict"],
547			print_results,
548			section,
549			log_printer),
550			arg_dict["local_result_dict"],
551			arg_dict["global_result_dict"],
552			arg_dict["file_dict"])
553			finally:
554			logger_thread.running = False
555
556			for runner in processes:
557			runner.join()
558

coala-analyzer / coala

Pull Request — master (#1170)

coalib.processes.autoapply_actions() C

Complexity

Size

Duplication

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like