autoapply_actions() - Code Metrics - Inspection of "Sils/cache" - coala-analyzer/coala - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2292)

by Lasse

created 2016-06-06 01:12 UTC

autoapply_actions() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	10
c	0
b	0
f	0
dl	0
loc	67
rs	4.3902

How to fix Long Method Complexity

import multiprocessing
import os
import platform
import queue
import subprocess
from itertools import chain

from coalib.collecting import Dependencies
from coalib.collecting.Collectors import collect_files
from coalib.misc.StringConverter import StringConverter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.BearRunning import run
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.processes.LogPrinterThread import LogPrinterThread
from coalib.results.Result import Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange
from coalib.settings.Setting import glob_list
from coalib.parsing.Globbing import fnmatch

ACTIONS = [ApplyPatchAction,
           PrintDebugMessageAction,
           ShowPatchAction]


def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    # cpu_count is not implemented for some CPU architectures/OSes
    except NotImplementedError:  # pragma: no cover
        return 2


def fill_queue(queue_fill, any_list):
    """
    Takes element from a list and populates a queue with those elements.

    :param queue_fill: The queue to be filled.
    :param any_list:   List containing the elements.
    """
    for elem in any_list:
        queue_fill.put(elem)


def get_running_processes(processes):
    return sum((1 if process.is_alive() else 0) for process in processes)


def create_process_group(command_array, **kwargs):
    if platform.system() == "Windows":  # pragma: no cover
        proc = subprocess.Popen(
            command_array,
            creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
            **kwargs)
    else:
        proc = subprocess.Popen(command_array,
                                preexec_fn=os.setsid,
                                **kwargs)
    return proc


def get_default_actions(section):
    """
    Parses the key ``default_actions`` in the given section.

    :param section:    The section where to parse from.
    :return:           A dict with the bearname as keys and their default
                       actions as values and another dict that contains bears
                       and invalid action names.
    """
    try:
        default_actions = dict(section["default_actions"])
    except IndexError:
        return {}, {}

    action_dict = {action.get_metadata().name: action for action in ACTIONS}
    invalid_action_set = default_actions.values() - action_dict.keys()
    invalid_actions = {}
    if len(invalid_action_set) != 0:
        invalid_actions = {
            bear: action
            for bear, action in default_actions.items()
            if action in invalid_action_set}
        for invalid in invalid_actions.keys():
            del default_actions[invalid]

    actions = {bearname: action_dict[action_name]
               for bearname, action_name in default_actions.items()}
    return actions, invalid_actions


def autoapply_actions(results,
                      file_dict,
                      file_diff_dict,
                      section,
                      log_printer):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """

    default_actions, invalid_actions = get_default_actions(section)

    for bearname, actionname in invalid_actions.items():
        log_printer.warn("Selected default action {!r} for bear {!r} does "
                         "not exist. Ignoring action.".format(actionname,
                                                              bearname))

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            action = default_actions[result.origin]
        except KeyError:
            for action_name in default_actions:
                if fnmatch(result.origin, action_name):
                    action = default_actions[action_name]
                    break
            else:
                not_processed_results.append(result)
                continue

        if not action.is_applicable(result, file_dict, file_diff_dict):
            log_printer.warn("Selected default action {!r} for bear {!r} is "
                             "not applicable. Action not applied.".format(
                                 action.get_metadata().name, result.origin))
            not_processed_results.append(result)
            continue

        try:
            action().apply_from_section(result,
                                        file_dict,
                                        file_diff_dict,
                                        section)
            log_printer.info("Applied {!r} on {} from {!r}.".format(
                action.get_metadata().name,
                result.location_repr(),
                result.origin))
        except Exception as ex:
            not_processed_results.append(result)
            log_printer.log_exception(
                "Failed to execute action {!r} with error: {}.".format(
                    action.get_metadata().name, ex),
                ex)
            log_printer.debug("-> for result " + repr(result) + ".")

    return not_processed_results


def check_result_ignore(result, ignore_ranges):
    """
    Determines if the result has to be ignored.

    :param result:        The result that needs to be checked.
    :param ignore_ranges: A list of tuples, each containing a list of lower
                          cased affected bearnames and a SourceRange to
                          ignore. If any of the bearname lists is empty, it
                          is considered an ignore range for all bears.
                          This may be a list of globbed bear wildcards.
    :return:              True if the result has to be ignored.
    """
    for bears, range in ignore_ranges:
        orig = result.origin.lower()
        if (result.overlaps(range) and
                (len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
            return True

    return False


def print_result(results,
                 file_dict,
                 retval,
                 print_results,
                 section,
                 log_printer,
                 file_diff_dict,
                 ignore_ranges):
    """
    Takes the results produced by each bear and gives them to the print_results
    method to present to the user.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param retval:         It is True if no results were yielded ever before.
                           If it is False this function will return False no
                           matter what happens. Else it depends on if this
                           invocation yields results.
    :param print_results:  A function that prints all given results appropriate
                           to the output medium.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param ignore_ranges:  A list of SourceRanges. Results that affect code in
                           any of those ranges will be ignored.
    :return:               Returns False if any results were yielded. Else
                           True.
    """
    min_severity_str = str(section.get('min_severity', 'INFO')).upper()
    min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
    results = list(filter(lambda result:
                          type(result) is Result and
                          result.severity >= min_severity and
                          not check_result_ignore(result, ignore_ranges),
                          results))

    if bool(section.get('autoapply', 'true')):
        patched_results = autoapply_actions(results,
                                            file_dict,
                                            file_diff_dict,
                                            section,
                                            log_printer)
    else:
        patched_results = results

    print_results(log_printer,
                  section,
                  patched_results,
                  file_dict,
                  file_diff_dict)
    return retval or len(results) > 0, patched_results


def get_file_dict(filename_list, log_printer):
    """
    Reads all files into a dictionary.

    :param filename_list: List of names of paths to files to get contents of.
    :param log_printer:   The logger which logs errors.
    :return:              Reads the content of each file into a dictionary
                          with filenames as keys.
    """
    file_dict = {}
    for filename in filename_list:
        try:
            with open(filename, "r", encoding="utf-8") as _file:
                file_dict[filename] = tuple(_file.readlines())
        except UnicodeDecodeError:
            log_printer.warn("Failed to read file '{}'. It seems to contain "
                             "non-unicode characters. Leaving it "
                             "out.".format(filename))
        except OSError as exception:  # pragma: no cover
            log_printer.log_exception("Failed to read file '{}' because of "
                                      "an unknown error. Leaving it "
                                      "out.".format(filename),
                                      exception,
                                      log_level=LOG_LEVEL.WARNING)

    log_printer.debug("Files that will be checked:\n" +
                      "\n".join(file_dict.keys()))
    return file_dict


def filter_raising_callables(it, exception, *args, **kwargs):
    """
    Filters all callable items inside the given iterator that raise the
    given exceptions.

    :param it:        The iterator to filter.
    :param exception: The (tuple of) exception(s) to filter for.
    :param args:      Positional arguments to pass to the callable.
    :param kwargs:    Keyword arguments to pass to the callable.
    """
    for elem in it:
        try:
            yield elem(*args, **kwargs)
        except exception:
            pass


def instantiate_bears(section,
                      local_bear_list,
                      global_bear_list,
                      file_dict,
                      message_queue):
    """
    Instantiates each bear with the arguments it needs.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bear classes to instantiate.
    :param global_bear_list: List of global bear classes to instantiate.
    :param file_dict:        Dictionary containing filenames and their
                             contents.
    :param message_queue:    Queue responsible to maintain the messages
                             delivered by the bears.
    :return:                 The local and global bear instance lists.
    """
    local_bear_list = [bear
                       for bear in filter_raising_callables(
                           local_bear_list,
                           RuntimeError,
                           section,
                           message_queue,
                           timeout=0.1)]

    global_bear_list = [bear
                        for bear in filter_raising_callables(
                            global_bear_list,
                            RuntimeError,
                            file_dict,
                            section,
                            message_queue,
                            timeout=0.1)]

    return local_bear_list, global_bear_list


def instantiate_processes(section,
                          local_bear_list,
                          global_bear_list,
                          job_count,
                          log_printer):
    """
    Instantiate the number of processes that will run bears which will be
    responsible for running bears in a multiprocessing environment.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bears belonging to the section.
    :param global_bear_list: List of global bears belonging to the section.
    :param job_count:        Max number of processes to create.
    :param log_printer:      The log printer to warn to.
    :return:                 A tuple containing a list of processes,
                             and the arguments passed to each process which are
                             the same for each object.
    """
    filename_list = collect_files(
        glob_list(section.get('files', "")),
        log_printer,
        ignored_file_paths=glob_list(section.get('ignore', "")),
        limit_file_paths=glob_list(section.get('limit_files', "")))
    file_dict = get_file_dict(filename_list, log_printer)

    manager = multiprocessing.Manager()
    global_bear_queue = multiprocessing.Queue()
    filename_queue = multiprocessing.Queue()
    local_result_dict = manager.dict()
    global_result_dict = manager.dict()
    message_queue = multiprocessing.Queue()
    control_queue = multiprocessing.Queue()

    bear_runner_args = {"file_name_queue": filename_queue,
                        "local_bear_list": local_bear_list,
                        "global_bear_list": global_bear_list,
                        "global_bear_queue": global_bear_queue,
                        "file_dict": file_dict,
                        "local_result_dict": local_result_dict,
                        "global_result_dict": global_result_dict,
                        "message_queue": message_queue,
                        "control_queue": control_queue,
                        "timeout": 0.1}

    local_bear_list[:], global_bear_list[:] = instantiate_bears(
        section,
        local_bear_list,
        global_bear_list,
        file_dict,
        message_queue)

    fill_queue(filename_queue, file_dict.keys())
    fill_queue(global_bear_queue, range(len(global_bear_list)))

    return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
             for i in range(job_count)],
            bear_runner_args)


def get_ignore_scope(line, keyword):
    """
    Retrieves the bears that are to be ignored defined in the given line.

    :param line:    The line containing the ignore declaration.
    :param keyword: The keyword that was found. Everything after the rightmost
                    occurrence of it will be considered for the scope.
    :return:        A list of lower cased bearnames or an empty list (-> "all")
    """
    toignore = line[line.rfind(keyword) + len(keyword):]
    if toignore.startswith("all"):
        return []
    else:
        return list(StringConverter(toignore, list_delimiters=', '))


def yield_ignore_ranges(file_dict):
    """
    Yields tuples of affected bears and a SourceRange that shall be ignored for
    those.

    :param file_dict: The file dictionary.
    """
    for filename, file in file_dict.items():
        start = None
        bears = []
        stop_ignoring = False
        for line_number, line in enumerate(file, start=1):
            # Before lowering all lines ever read, first look for the biggest
            # common substring.
            if 'gnor' in line:
                line = line.lower()
                if "start ignoring " in line:
                    start = line_number
                    bears = get_ignore_scope(line, "start ignoring ")
                elif "stop ignoring" in line:
                    stop_ignoring = True
                    if start:
                        yield (bears,
                               SourceRange.from_values(
                                   filename,
                                   start,
                                   1,
                                   line_number,
                                   len(file[line_number-1])))
                elif "ignore " in line:
                    yield (get_ignore_scope(line, "ignore "),
                           SourceRange.from_values(filename,
                                                   line_number,
                                                   1,
                                                   line_number+1,
                                                   len(file[line_number])))
        if stop_ignoring is False and start is not None:
            yield (bears,
                   SourceRange.from_values(filename,
                                           start,
                                           1,
                                           len(file),
                                           len(file[-1])))


def process_queues(processes,
                   control_queue,
                   local_result_dict,
                   global_result_dict,
                   file_dict,
                   print_results,
                   section,
                   log_printer):
    """
    Iterate the control queue and send the results recieved to the print_result
    method so that they can be presented to the user.

    :param processes:          List of processes which can be used to run
                               Bears.
    :param control_queue:      Containing control elements that indicate
                               whether there is a result available and which
                               bear it belongs to.
    :param local_result_dict:  Dictionary containing results respective to
                               local bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param global_result_dict: Dictionary containing results respective to
                               global bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param file_dict:          Dictionary containing file contents with
                               filename as keys.
    :param print_results:      Prints all given results appropriate to the
                               output medium.
    :return:                   Return True if all bears execute succesfully and
                               Results were delivered to the user. Else False.
    """
    file_diff_dict = {}
    retval = False
    # Number of processes working on local/global bears. They are count down
    # when the last queue element of that process is processed which may be
    # *after* the process has ended!
    local_processes = len(processes)
    global_processes = len(processes)
    global_result_buffer = []
    ignore_ranges = list(yield_ignore_ranges(file_dict))

    # One process is the logger thread
    while local_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
                local_processes -= 1
            elif control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED:
                global_processes -= 1
            elif control_elem == CONTROL_ELEMENT.LOCAL:
                assert local_processes != 0
                retval, res = print_result(local_result_dict[index],
                                           file_dict,
                                           retval,
                                           print_results,
                                           section,
                                           log_printer,
                                           file_diff_dict,
                                           ignore_ranges)
                local_result_dict[index] = res
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL
                global_result_buffer.append(index)
        except queue.Empty:
            if get_running_processes(processes) < 2:  # pragma: no cover
                # Recover silently, those branches are only
                # nondeterministically covered.
                break

    # Flush global result buffer
    for elem in global_result_buffer:
        retval, res = print_result(global_result_dict[elem],
                                   file_dict,
                                   retval,
                                   print_results,
                                   section,
                                   log_printer,
                                   file_diff_dict,
                                   ignore_ranges)
        global_result_dict[elem] = res

    # One process is the logger thread
    while global_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.GLOBAL:
                retval, res = print_result(global_result_dict[index],
                                           file_dict,
                                           retval,
                                           print_results,
                                           section,
                                           log_printer,
                                           file_diff_dict,
                                           ignore_ranges)
                global_result_dict[index] = res
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
                global_processes -= 1
        except queue.Empty:
            if get_running_processes(processes) < 2:  # pragma: no cover
                # Recover silently, those branches are only
                # nondeterministically covered.
                break

    return retval


def simplify_section_result(section_result):
    """
    Takes in a section's result from ``execute_section`` and simplifies it
    for easy usage in other functions.

    :param section_result: The result of a section which was executed.
    :return:               Tuple containing:
                            - bool - True if results were yielded
                            - bool - True if unfixed results were yielded
                            - list - Results from all bears (local and global)
    """
    section_yielded_result = section_result[0]
    results_for_section = []
    for value in chain(section_result[1].values(),
                       section_result[2].values()):
        if value is None:
            continue

        for result in value:
            results_for_section.append(result)
    section_yielded_unfixed_results = len(results_for_section) > 0

    return (section_yielded_result,
            section_yielded_unfixed_results,
            results_for_section)


def execute_section(section,
                    global_bear_list,
                    local_bear_list,
                    print_results,
                    log_printer):
    """
    Executes the section with the given bears.

    The execute_section method does the following things:

    1. Prepare a Process
       -  Load files
       -  Create queues
    2. Spawn up one or more Processes
    3. Output results from the Processes
    4. Join all processes

    :param section:          The section to execute.
    :param global_bear_list: List of global bears belonging to the section.
    :param local_bear_list:  List of local bears belonging to the section.
    :param print_results:    Prints all given results appropriate to the
                             output medium.
    :param log_printer:      The log_printer to warn to.
    :return:                 Tuple containing a bool (True if results were
                             yielded, False otherwise), a Manager.dict
                             containing all local results(filenames are key)
                             and a Manager.dict containing all global bear
                             results (bear names are key) as well as the
                             file dictionary.
    """
    local_bear_list = Dependencies.resolve(local_bear_list)
    global_bear_list = Dependencies.resolve(global_bear_list)

    try:
        running_processes = int(section['jobs'])
    except ValueError:
        log_printer.warn("Unable to convert setting 'jobs' into a number. "
                         "Falling back to CPU count.")
        running_processes = get_cpu_count()
    except IndexError:
        running_processes = get_cpu_count()

    processes, arg_dict = instantiate_processes(section,
                                                local_bear_list,
                                                global_bear_list,
                                                running_processes,
                                                log_printer)

    logger_thread = LogPrinterThread(arg_dict["message_queue"],
                                     log_printer)
    # Start and join the logger thread along with the processes to run bears
    processes.append(logger_thread)

    for runner in processes:
        runner.start()

    try:
        return (process_queues(processes,
                               arg_dict["control_queue"],
                               arg_dict["local_result_dict"],
                               arg_dict["global_result_dict"],
                               arg_dict["file_dict"],
                               print_results,
                               section,
                               log_printer),
                arg_dict["local_result_dict"],
                arg_dict["global_result_dict"],
                arg_dict["file_dict"])
    finally:
        logger_thread.running = False

        for runner in processes:
            runner.join()


1			import multiprocessing
2			import os
3			import platform
4			import queue
5			import subprocess
6			from itertools import chain
7
8			from coalib.collecting import Dependencies
9			from coalib.collecting.Collectors import collect_files
10			from coalib.misc.StringConverter import StringConverter
11			from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
12			from coalib.processes.BearRunning import run
13			from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
14			from coalib.processes.LogPrinterThread import LogPrinterThread
15			from coalib.results.Result import Result
16			from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
17			from coalib.results.result_actions.PrintDebugMessageAction import (
18			PrintDebugMessageAction)
19			from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
20			from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
21			from coalib.results.SourceRange import SourceRange
22			from coalib.settings.Setting import glob_list
23			from coalib.parsing.Globbing import fnmatch
24
25			ACTIONS = [ApplyPatchAction,
26			PrintDebugMessageAction,
27			ShowPatchAction]
28
29
30			def get_cpu_count():
31			try:
32			return multiprocessing.cpu_count()
33			# cpu_count is not implemented for some CPU architectures/OSes
34			except NotImplementedError: # pragma: no cover
35			return 2
36
37
38			def fill_queue(queue_fill, any_list):
39			"""
40			Takes element from a list and populates a queue with those elements.
41
42			:param queue_fill: The queue to be filled.
43			:param any_list: List containing the elements.
44			"""
45			for elem in any_list:
46			queue_fill.put(elem)
47
48
49			def get_running_processes(processes):
50			return sum((1 if process.is_alive() else 0) for process in processes)
51
52
53			def create_process_group(command_array, **kwargs):
54			if platform.system() == "Windows": # pragma: no cover
55			proc = subprocess.Popen(
56			command_array,
57			creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
58			**kwargs)
59			else:
60			proc = subprocess.Popen(command_array,
61			preexec_fn=os.setsid,
62			**kwargs)
63			return proc
64
65
66			def get_default_actions(section):
67			"""
68			Parses the key ``default_actions`` in the given section.
69
70			:param section: The section where to parse from.
71			:return: A dict with the bearname as keys and their default
72			actions as values and another dict that contains bears
73			and invalid action names.
74			"""
75			try:
76			default_actions = dict(section["default_actions"])
77			except IndexError:
78			return {}, {}
79
80			action_dict = {action.get_metadata().name: action for action in ACTIONS}
81			invalid_action_set = default_actions.values() - action_dict.keys()
82			invalid_actions = {}
83			if len(invalid_action_set) != 0:
84			invalid_actions = {
85			bear: action
86			for bear, action in default_actions.items()
87			if action in invalid_action_set}
88			for invalid in invalid_actions.keys():
89			del default_actions[invalid]
90
91			actions = {bearname: action_dict[action_name]
92			for bearname, action_name in default_actions.items()}
93			return actions, invalid_actions
94
95
96			def autoapply_actions(results,
97			file_dict,
98			file_diff_dict,
99			section,
100			log_printer):
101			"""
102			Auto-applies actions like defined in the given section.
103
104			:param results: A list of results.
105			:param file_dict: A dictionary containing the name of files and its
106			contents.
107			:param file_diff_dict: A dictionary that contains filenames as keys and
108			diff objects as values.
109			:param section: The section.
110			:param log_printer: A log printer instance to log messages on.
111			:return: A list of unprocessed results.
112			"""
113
114			default_actions, invalid_actions = get_default_actions(section)
115
116			for bearname, actionname in invalid_actions.items():
117			log_printer.warn("Selected default action {!r} for bear {!r} does "
118			"not exist. Ignoring action.".format(actionname,
119			bearname))
120
121			if len(default_actions) == 0:
122			# There's nothing to auto-apply.
123			return results
124
125			not_processed_results = []
126			for result in results:
127			try:
128			action = default_actions[result.origin]
129			except KeyError:
130			for action_name in default_actions:
131			if fnmatch(result.origin, action_name):
132			action = default_actions[action_name]
133			break
134			else:
135			not_processed_results.append(result)
136			continue
137
138			if not action.is_applicable(result, file_dict, file_diff_dict):
139			log_printer.warn("Selected default action {!r} for bear {!r} is "
140			"not applicable. Action not applied.".format(
141			action.get_metadata().name, result.origin))
142			not_processed_results.append(result)
143			continue
144
145			try:
146			action().apply_from_section(result,
147			file_dict,
148			file_diff_dict,
149			section)
150			log_printer.info("Applied {!r} on {} from {!r}.".format(
151			action.get_metadata().name,
152			result.location_repr(),
153			result.origin))
154			except Exception as ex:
155			not_processed_results.append(result)
156			log_printer.log_exception(
157			"Failed to execute action {!r} with error: {}.".format(
158			action.get_metadata().name, ex),
159			ex)
160			log_printer.debug("-> for result " + repr(result) + ".")
161
162			return not_processed_results
163
164
165			def check_result_ignore(result, ignore_ranges):
166			"""
167			Determines if the result has to be ignored.
168
169			:param result: The result that needs to be checked.
170			:param ignore_ranges: A list of tuples, each containing a list of lower
171			cased affected bearnames and a SourceRange to
172			ignore. If any of the bearname lists is empty, it
173			is considered an ignore range for all bears.
174			This may be a list of globbed bear wildcards.
175			:return: True if the result has to be ignored.
176			"""
177			for bears, range in ignore_ranges:
178			orig = result.origin.lower()
179			if (result.overlaps(range) and
180			(len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
181			return True
182
183			return False
184
185
186			def print_result(results,
187			file_dict,
188			retval,
189			print_results,
190			section,
191			log_printer,
192			file_diff_dict,
193			ignore_ranges):
194			"""
195			Takes the results produced by each bear and gives them to the print_results
196			method to present to the user.
197
198			:param results: A list of results.
199			:param file_dict: A dictionary containing the name of files and its
200			contents.
201			:param retval: It is True if no results were yielded ever before.
202			If it is False this function will return False no
203			matter what happens. Else it depends on if this
204			invocation yields results.
205			:param print_results: A function that prints all given results appropriate
206			to the output medium.
207			:param file_diff_dict: A dictionary that contains filenames as keys and
208			diff objects as values.
209			:param ignore_ranges: A list of SourceRanges. Results that affect code in
210			any of those ranges will be ignored.
211			:return: Returns False if any results were yielded. Else
212			True.
213			"""
214			min_severity_str = str(section.get('min_severity', 'INFO')).upper()
215			min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
216			results = list(filter(lambda result:
217			type(result) is Result and
218			result.severity >= min_severity and
219			not check_result_ignore(result, ignore_ranges),
220			results))
221
222			if bool(section.get('autoapply', 'true')):
223			patched_results = autoapply_actions(results,
224			file_dict,
225			file_diff_dict,
226			section,
227			log_printer)
228			else:
229			patched_results = results
230
231			print_results(log_printer,
232			section,
233			patched_results,
234			file_dict,
235			file_diff_dict)
236			return retval or len(results) > 0, patched_results
237
238
239			def get_file_dict(filename_list, log_printer):
240			"""
241			Reads all files into a dictionary.
242
243			:param filename_list: List of names of paths to files to get contents of.
244			:param log_printer: The logger which logs errors.
245			:return: Reads the content of each file into a dictionary
246			with filenames as keys.
247			"""
248			file_dict = {}
249			for filename in filename_list:
250			try:
251			with open(filename, "r", encoding="utf-8") as _file:
252			file_dict[filename] = tuple(_file.readlines())
253			except UnicodeDecodeError:
254			log_printer.warn("Failed to read file '{}'. It seems to contain "
255			"non-unicode characters. Leaving it "
256			"out.".format(filename))
257			except OSError as exception: # pragma: no cover
258			log_printer.log_exception("Failed to read file '{}' because of "
259			"an unknown error. Leaving it "
260			"out.".format(filename),
261			exception,
262			log_level=LOG_LEVEL.WARNING)
263
264			log_printer.debug("Files that will be checked:\n" +
265			"\n".join(file_dict.keys()))
266			return file_dict
267
268
269			def filter_raising_callables(it, exception, args, *kwargs):
270			"""
271			Filters all callable items inside the given iterator that raise the
272			given exceptions.
273
274			:param it: The iterator to filter.
275			:param exception: The (tuple of) exception(s) to filter for.
276			:param args: Positional arguments to pass to the callable.
277			:param kwargs: Keyword arguments to pass to the callable.
278			"""
279			for elem in it:
280			try:
281			yield elem(args, *kwargs)
282			except exception:
283			pass
284
285
286			def instantiate_bears(section,
287			local_bear_list,
288			global_bear_list,
289			file_dict,
290			message_queue):
291			"""
292			Instantiates each bear with the arguments it needs.
293
294			:param section: The section the bears belong to.
295			:param local_bear_list: List of local bear classes to instantiate.
296			:param global_bear_list: List of global bear classes to instantiate.
297			:param file_dict: Dictionary containing filenames and their
298			contents.
299			:param message_queue: Queue responsible to maintain the messages
300			delivered by the bears.
301			:return: The local and global bear instance lists.
302			"""
303			local_bear_list = [bear
304			for bear in filter_raising_callables(
305			local_bear_list,
306			RuntimeError,
307			section,
308			message_queue,
309			timeout=0.1)]
310
311			global_bear_list = [bear
312			for bear in filter_raising_callables(
313			global_bear_list,
314			RuntimeError,
315			file_dict,
316			section,
317			message_queue,
318			timeout=0.1)]
319
320			return local_bear_list, global_bear_list
321
322
323			def instantiate_processes(section,
324			local_bear_list,
325			global_bear_list,
326			job_count,
327			log_printer):
328			"""
329			Instantiate the number of processes that will run bears which will be
330			responsible for running bears in a multiprocessing environment.
331
332			:param section: The section the bears belong to.
333			:param local_bear_list: List of local bears belonging to the section.
334			:param global_bear_list: List of global bears belonging to the section.
335			:param job_count: Max number of processes to create.
336			:param log_printer: The log printer to warn to.
337			:return: A tuple containing a list of processes,
338			and the arguments passed to each process which are
339			the same for each object.
340			"""
341			filename_list = collect_files(
342			glob_list(section.get('files', "")),
343			log_printer,
344			ignored_file_paths=glob_list(section.get('ignore', "")),
345			limit_file_paths=glob_list(section.get('limit_files', "")))
346			file_dict = get_file_dict(filename_list, log_printer)
347
348			manager = multiprocessing.Manager()
349			global_bear_queue = multiprocessing.Queue()
350			filename_queue = multiprocessing.Queue()
351			local_result_dict = manager.dict()
352			global_result_dict = manager.dict()
353			message_queue = multiprocessing.Queue()
354			control_queue = multiprocessing.Queue()
355
356			bear_runner_args = {"file_name_queue": filename_queue,
357			"local_bear_list": local_bear_list,
358			"global_bear_list": global_bear_list,
359			"global_bear_queue": global_bear_queue,
360			"file_dict": file_dict,
361			"local_result_dict": local_result_dict,
362			"global_result_dict": global_result_dict,
363			"message_queue": message_queue,
364			"control_queue": control_queue,
365			"timeout": 0.1}
366
367			local_bear_list[:], global_bear_list[:] = instantiate_bears(
368			section,
369			local_bear_list,
370			global_bear_list,
371			file_dict,
372			message_queue)
373
374			fill_queue(filename_queue, file_dict.keys())
375			fill_queue(global_bear_queue, range(len(global_bear_list)))
376
377			return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
378			for i in range(job_count)],
379			bear_runner_args)
380
381
382			def get_ignore_scope(line, keyword):
383			"""
384			Retrieves the bears that are to be ignored defined in the given line.
385
386			:param line: The line containing the ignore declaration.
387			:param keyword: The keyword that was found. Everything after the rightmost
388			occurrence of it will be considered for the scope.
389			:return: A list of lower cased bearnames or an empty list (-> "all")
390			"""
391			toignore = line[line.rfind(keyword) + len(keyword):]
392			if toignore.startswith("all"):
393			return []
394			else:
395			return list(StringConverter(toignore, list_delimiters=', '))
396
397
398			def yield_ignore_ranges(file_dict):
399			"""
400			Yields tuples of affected bears and a SourceRange that shall be ignored for
401			those.
402
403			:param file_dict: The file dictionary.
404			"""
405			for filename, file in file_dict.items():
406			start = None
407			bears = []
408			stop_ignoring = False
409			for line_number, line in enumerate(file, start=1):
410			# Before lowering all lines ever read, first look for the biggest
411			# common substring.
412			if 'gnor' in line:
413			line = line.lower()
414			if "start ignoring " in line:
415			start = line_number
416			bears = get_ignore_scope(line, "start ignoring ")
417			elif "stop ignoring" in line:
418			stop_ignoring = True
419			if start:
420			yield (bears,
421			SourceRange.from_values(
422			filename,
423			start,
424			1,
425			line_number,
426			len(file[line_number-1])))
427			elif "ignore " in line:
428			yield (get_ignore_scope(line, "ignore "),
429			SourceRange.from_values(filename,
430			line_number,
431			1,
432			line_number+1,
433			len(file[line_number])))
434			if stop_ignoring is False and start is not None:
435			yield (bears,
436			SourceRange.from_values(filename,
437			start,
438			1,
439			len(file),
440			len(file[-1])))
441
442
443			def process_queues(processes,
444			control_queue,
445			local_result_dict,
446			global_result_dict,
447			file_dict,
448			print_results,
449			section,
450			log_printer):
451			"""
452			Iterate the control queue and send the results recieved to the print_result
453			method so that they can be presented to the user.
454
455			:param processes: List of processes which can be used to run
456			Bears.
457			:param control_queue: Containing control elements that indicate
458			whether there is a result available and which
459			bear it belongs to.
460			:param local_result_dict: Dictionary containing results respective to
461			local bears. It is modified by the processes
462			i.e. results are added to it by multiple
463			processes.
464			:param global_result_dict: Dictionary containing results respective to
465			global bears. It is modified by the processes
466			i.e. results are added to it by multiple
467			processes.
468			:param file_dict: Dictionary containing file contents with
469			filename as keys.
470			:param print_results: Prints all given results appropriate to the
471			output medium.
472			:return: Return True if all bears execute succesfully and
473			Results were delivered to the user. Else False.
474			"""
475			file_diff_dict = {}
476			retval = False
477			# Number of processes working on local/global bears. They are count down
478			# when the last queue element of that process is processed which may be
479			# after the process has ended!
480			local_processes = len(processes)
481			global_processes = len(processes)
482			global_result_buffer = []
483			ignore_ranges = list(yield_ignore_ranges(file_dict))
484
485			# One process is the logger thread
486			while local_processes > 1:
487			try:
488			control_elem, index = control_queue.get(timeout=0.1)
489
490			if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
491			local_processes -= 1
492			elif control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED:
493			global_processes -= 1
494			elif control_elem == CONTROL_ELEMENT.LOCAL:
495			assert local_processes != 0
496			retval, res = print_result(local_result_dict[index],
497			file_dict,
498			retval,
499			print_results,
500			section,
501			log_printer,
502			file_diff_dict,
503			ignore_ranges)
504			local_result_dict[index] = res
505			else:
506			assert control_elem == CONTROL_ELEMENT.GLOBAL
507			global_result_buffer.append(index)
508			except queue.Empty:
509			if get_running_processes(processes) < 2: # pragma: no cover
510			# Recover silently, those branches are only
511			# nondeterministically covered.
512			break
513
514			# Flush global result buffer
515			for elem in global_result_buffer:
516			retval, res = print_result(global_result_dict[elem],
517			file_dict,
518			retval,
519			print_results,
520			section,
521			log_printer,
522			file_diff_dict,
523			ignore_ranges)
524			global_result_dict[elem] = res
525
526			# One process is the logger thread
527			while global_processes > 1:
528			try:
529			control_elem, index = control_queue.get(timeout=0.1)
530
531			if control_elem == CONTROL_ELEMENT.GLOBAL:
532			retval, res = print_result(global_result_dict[index],
533			file_dict,
534			retval,
535			print_results,
536			section,
537			log_printer,
538			file_diff_dict,
539			ignore_ranges)
540			global_result_dict[index] = res
541			else:
542			assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
543			global_processes -= 1
544			except queue.Empty:
545			if get_running_processes(processes) < 2: # pragma: no cover
546			# Recover silently, those branches are only
547			# nondeterministically covered.
548			break
549
550			return retval
551
552
553			def simplify_section_result(section_result):
554			"""
555			Takes in a section's result from ``execute_section`` and simplifies it
556			for easy usage in other functions.
557
558			:param section_result: The result of a section which was executed.
559			:return: Tuple containing:
560			- bool - True if results were yielded
561			- bool - True if unfixed results were yielded
562			- list - Results from all bears (local and global)
563			"""
564			section_yielded_result = section_result[0]
565			results_for_section = []
566			for value in chain(section_result[1].values(),
567			section_result[2].values()):
568			if value is None:
569			continue
570
571			for result in value:
572			results_for_section.append(result)
573			section_yielded_unfixed_results = len(results_for_section) > 0
574
575			return (section_yielded_result,
576			section_yielded_unfixed_results,
577			results_for_section)
578
579
580			def execute_section(section,
581			global_bear_list,
582			local_bear_list,
583			print_results,
584			log_printer):
585			"""
586			Executes the section with the given bears.
587
588			The execute_section method does the following things:
589
590			1. Prepare a Process
591			- Load files
592			- Create queues
593			2. Spawn up one or more Processes
594			3. Output results from the Processes
595			4. Join all processes
596
597			:param section: The section to execute.
598			:param global_bear_list: List of global bears belonging to the section.
599			:param local_bear_list: List of local bears belonging to the section.
600			:param print_results: Prints all given results appropriate to the
601			output medium.
602			:param log_printer: The log_printer to warn to.
603			:return: Tuple containing a bool (True if results were
604			yielded, False otherwise), a Manager.dict
605			containing all local results(filenames are key)
606			and a Manager.dict containing all global bear
607			results (bear names are key) as well as the
608			file dictionary.
609			"""
610			local_bear_list = Dependencies.resolve(local_bear_list)
611			global_bear_list = Dependencies.resolve(global_bear_list)
612
613			try:
614			running_processes = int(section['jobs'])
615			except ValueError:
616			log_printer.warn("Unable to convert setting 'jobs' into a number. "
617			"Falling back to CPU count.")
618			running_processes = get_cpu_count()
619			except IndexError:
620			running_processes = get_cpu_count()
621
622			processes, arg_dict = instantiate_processes(section,
623			local_bear_list,
624			global_bear_list,
625			running_processes,
626			log_printer)
627
628			logger_thread = LogPrinterThread(arg_dict["message_queue"],
629			log_printer)
630			# Start and join the logger thread along with the processes to run bears
631			processes.append(logger_thread)
632
633			for runner in processes:
634			runner.start()
635
636			try:
637			return (process_queues(processes,
638			arg_dict["control_queue"],
639			arg_dict["local_result_dict"],
640			arg_dict["global_result_dict"],
641			arg_dict["file_dict"],
642			print_results,
643			section,
644			log_printer),
645			arg_dict["local_result_dict"],
646			arg_dict["global_result_dict"],
647			arg_dict["file_dict"])
648			finally:
649			logger_thread.running = False
650
651			for runner in processes:
652			runner.join()
653

coala-analyzer / coala

Pull Request — master (#2292)

autoapply_actions() D

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like