autoapply_actions() - Code Metrics - Inspection of "Sils/cache" - coala-analyzer/coala - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2292)

by Lasse

created 2016-06-06 02:08 UTC

autoapply_actions() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	10
dl	0
loc	68
rs	4.3373
c	0
b	0
f	0

How to fix Long Method Complexity

import multiprocessing
import os
import platform
import queue
import subprocess
from itertools import chain

from coalib.collecting import Dependencies
from coalib.collecting.Collectors import collect_files
from coalib.misc.StringConverter import StringConverter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.processes.BearRunning import run
from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
from coalib.processes.LogPrinterThread import LogPrinterThread
from coalib.results.Result import Result
from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
from coalib.results.result_actions.PrintDebugMessageAction import (
    PrintDebugMessageAction)
from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
from coalib.results.SourceRange import SourceRange
from coalib.settings.Setting import glob_list
from coalib.parsing.Globbing import fnmatch

ACTIONS = [ApplyPatchAction,
           PrintDebugMessageAction,
           ShowPatchAction]


def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    # cpu_count is not implemented for some CPU architectures/OSes
    except NotImplementedError:  # pragma: no cover
        return 2


def fill_queue(queue_fill, any_list):
    """
    Takes element from a list and populates a queue with those elements.

    :param queue_fill: The queue to be filled.
    :param any_list:   List containing the elements.
    """
    for elem in any_list:
        queue_fill.put(elem)


def get_running_processes(processes):
    return sum((1 if process.is_alive() else 0) for process in processes)


def create_process_group(command_array, **kwargs):
    if platform.system() == "Windows":  # pragma: no cover
        proc = subprocess.Popen(
            command_array,
            creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
            **kwargs)
    else:
        proc = subprocess.Popen(command_array,
                                preexec_fn=os.setsid,
                                **kwargs)
    return proc


def get_default_actions(section):
    """
    Parses the key ``default_actions`` in the given section.

    :param section:    The section where to parse from.
    :return:           A dict with the bearname as keys and their default
                       actions as values and another dict that contains bears
                       and invalid action names.
    """
    try:
        default_actions = dict(section["default_actions"])
    except IndexError:
        return {}, {}

    action_dict = {action.get_metadata().name: action for action in ACTIONS}
    invalid_action_set = default_actions.values() - action_dict.keys()
    invalid_actions = {}
    if len(invalid_action_set) != 0:
        invalid_actions = {
            bear: action
            for bear, action in default_actions.items()
            if action in invalid_action_set}
        for invalid in invalid_actions.keys():
            del default_actions[invalid]

    actions = {bearname: action_dict[action_name]
               for bearname, action_name in default_actions.items()}
    return actions, invalid_actions


def autoapply_actions(results,
                      file_dict,
                      file_diff_dict,
                      section,
                      log_printer):
    """
    Auto-applies actions like defined in the given section.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param section:        The section.
    :param log_printer:    A log printer instance to log messages on.
    :return:               A list of unprocessed results.
    """

    default_actions, invalid_actions = get_default_actions(section)

    for bearname, actionname in invalid_actions.items():
        log_printer.warn("Selected default action {!r} for bear {!r} does "
                         "not exist. Ignoring action.".format(actionname,
                                                              bearname))

    if len(default_actions) == 0:
        # There's nothing to auto-apply.
        return results

    not_processed_results = []
    for result in results:
        try:
            # Match full bear names deterministically, prioritized!
            action = default_actions[result.origin]
        except KeyError:
            for bear_glob in default_actions:
                if fnmatch(result.origin, bear_glob):
                    action = default_actions[bear_glob]
                    break
            else:
                not_processed_results.append(result)
                continue

        if not action.is_applicable(result, file_dict, file_diff_dict):
            log_printer.warn("Selected default action {!r} for bear {!r} is "
                             "not applicable. Action not applied.".format(
                                 action.get_metadata().name, result.origin))
            not_processed_results.append(result)
            continue

        try:
            action().apply_from_section(result,
                                        file_dict,
                                        file_diff_dict,
                                        section)
            log_printer.info("Applied {!r} on {} from {!r}.".format(
                action.get_metadata().name,
                result.location_repr(),
                result.origin))
        except Exception as ex:
            not_processed_results.append(result)
            log_printer.log_exception(
                "Failed to execute action {!r} with error: {}.".format(
                    action.get_metadata().name, ex),
                ex)
            log_printer.debug("-> for result " + repr(result) + ".")

    return not_processed_results


def check_result_ignore(result, ignore_ranges):
    """
    Determines if the result has to be ignored.

    :param result:        The result that needs to be checked.
    :param ignore_ranges: A list of tuples, each containing a list of lower
                          cased affected bearnames and a SourceRange to
                          ignore. If any of the bearname lists is empty, it
                          is considered an ignore range for all bears.
                          This may be a list of globbed bear wildcards.
    :return:              True if the result has to be ignored.
    """
    for bears, range in ignore_ranges:
        orig = result.origin.lower()
        if (result.overlaps(range) and
                (len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
            return True

    return False


def print_result(results,
                 file_dict,
                 retval,
                 print_results,
                 section,
                 log_printer,
                 file_diff_dict,
                 ignore_ranges):
    """
    Takes the results produced by each bear and gives them to the print_results
    method to present to the user.

    :param results:        A list of results.
    :param file_dict:      A dictionary containing the name of files and its
                           contents.
    :param retval:         It is True if no results were yielded ever before.
                           If it is False this function will return False no
                           matter what happens. Else it depends on if this
                           invocation yields results.
    :param print_results:  A function that prints all given results appropriate
                           to the output medium.
    :param file_diff_dict: A dictionary that contains filenames as keys and
                           diff objects as values.
    :param ignore_ranges:  A list of SourceRanges. Results that affect code in
                           any of those ranges will be ignored.
    :return:               Returns False if any results were yielded. Else
                           True.
    """
    min_severity_str = str(section.get('min_severity', 'INFO')).upper()
    min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
    results = list(filter(lambda result:
                          type(result) is Result and
                          result.severity >= min_severity and
                          not check_result_ignore(result, ignore_ranges),
                          results))

    if bool(section.get('autoapply', 'true')):
        patched_results = autoapply_actions(results,
                                            file_dict,
                                            file_diff_dict,
                                            section,
                                            log_printer)
    else:
        patched_results = results

    print_results(log_printer,
                  section,
                  patched_results,
                  file_dict,
                  file_diff_dict)
    return retval or len(results) > 0, patched_results


def get_file_dict(filename_list, log_printer):
    """
    Reads all files into a dictionary.

    :param filename_list: List of names of paths to files to get contents of.
    :param log_printer:   The logger which logs errors.
    :return:              Reads the content of each file into a dictionary
                          with filenames as keys.
    """
    file_dict = {}
    for filename in filename_list:
        try:
            with open(filename, "r", encoding="utf-8") as _file:
                file_dict[filename] = tuple(_file.readlines())
        except UnicodeDecodeError:
            log_printer.warn("Failed to read file '{}'. It seems to contain "
                             "non-unicode characters. Leaving it "
                             "out.".format(filename))
        except OSError as exception:  # pragma: no cover
            log_printer.log_exception("Failed to read file '{}' because of "
                                      "an unknown error. Leaving it "
                                      "out.".format(filename),
                                      exception,
                                      log_level=LOG_LEVEL.WARNING)

    log_printer.debug("Files that will be checked:\n" +
                      "\n".join(file_dict.keys()))
    return file_dict


def filter_raising_callables(it, exception, *args, **kwargs):
    """
    Filters all callable items inside the given iterator that raise the
    given exceptions.

    :param it:        The iterator to filter.
    :param exception: The (tuple of) exception(s) to filter for.
    :param args:      Positional arguments to pass to the callable.
    :param kwargs:    Keyword arguments to pass to the callable.
    """
    for elem in it:
        try:
            yield elem(*args, **kwargs)
        except exception:
            pass


def instantiate_bears(section,
                      local_bear_list,
                      global_bear_list,
                      file_dict,
                      message_queue):
    """
    Instantiates each bear with the arguments it needs.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bear classes to instantiate.
    :param global_bear_list: List of global bear classes to instantiate.
    :param file_dict:        Dictionary containing filenames and their
                             contents.
    :param message_queue:    Queue responsible to maintain the messages
                             delivered by the bears.
    :return:                 The local and global bear instance lists.
    """
    local_bear_list = [bear
                       for bear in filter_raising_callables(
                           local_bear_list,
                           RuntimeError,
                           section,
                           message_queue,
                           timeout=0.1)]

    global_bear_list = [bear
                        for bear in filter_raising_callables(
                            global_bear_list,
                            RuntimeError,
                            file_dict,
                            section,
                            message_queue,
                            timeout=0.1)]

    return local_bear_list, global_bear_list


def instantiate_processes(section,
                          local_bear_list,
                          global_bear_list,
                          job_count,
                          log_printer):
    """
    Instantiate the number of processes that will run bears which will be
    responsible for running bears in a multiprocessing environment.

    :param section:          The section the bears belong to.
    :param local_bear_list:  List of local bears belonging to the section.
    :param global_bear_list: List of global bears belonging to the section.
    :param job_count:        Max number of processes to create.
    :param log_printer:      The log printer to warn to.
    :return:                 A tuple containing a list of processes,
                             and the arguments passed to each process which are
                             the same for each object.
    """
    filename_list = collect_files(
        glob_list(section.get('files', "")),
        log_printer,
        ignored_file_paths=glob_list(section.get('ignore', "")),
        limit_file_paths=glob_list(section.get('limit_files', "")))
    file_dict = get_file_dict(filename_list, log_printer)

    manager = multiprocessing.Manager()
    global_bear_queue = multiprocessing.Queue()
    filename_queue = multiprocessing.Queue()
    local_result_dict = manager.dict()
    global_result_dict = manager.dict()
    message_queue = multiprocessing.Queue()
    control_queue = multiprocessing.Queue()

    bear_runner_args = {"file_name_queue": filename_queue,
                        "local_bear_list": local_bear_list,
                        "global_bear_list": global_bear_list,
                        "global_bear_queue": global_bear_queue,
                        "file_dict": file_dict,
                        "local_result_dict": local_result_dict,
                        "global_result_dict": global_result_dict,
                        "message_queue": message_queue,
                        "control_queue": control_queue,
                        "timeout": 0.1}

    local_bear_list[:], global_bear_list[:] = instantiate_bears(
        section,
        local_bear_list,
        global_bear_list,
        file_dict,
        message_queue)

    fill_queue(filename_queue, file_dict.keys())
    fill_queue(global_bear_queue, range(len(global_bear_list)))

    return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
             for i in range(job_count)],
            bear_runner_args)


def get_ignore_scope(line, keyword):
    """
    Retrieves the bears that are to be ignored defined in the given line.

    :param line:    The line containing the ignore declaration.
    :param keyword: The keyword that was found. Everything after the rightmost
                    occurrence of it will be considered for the scope.
    :return:        A list of lower cased bearnames or an empty list (-> "all")
    """
    toignore = line[line.rfind(keyword) + len(keyword):]
    if toignore.startswith("all"):
        return []
    else:
        return list(StringConverter(toignore, list_delimiters=', '))


def yield_ignore_ranges(file_dict):
    """
    Yields tuples of affected bears and a SourceRange that shall be ignored for
    those.

    :param file_dict: The file dictionary.
    """
    for filename, file in file_dict.items():
        start = None
        bears = []
        stop_ignoring = False
        for line_number, line in enumerate(file, start=1):
            # Before lowering all lines ever read, first look for the biggest
            # common substring, case sensitive: I*gnor*e, start i*gnor*ing.
            if 'gnor' in line:
                line = line.lower()
                if "start ignoring " in line:
                    start = line_number
                    bears = get_ignore_scope(line, "start ignoring ")
                elif "stop ignoring" in line:
                    stop_ignoring = True
                    if start:
                        yield (bears,
                               SourceRange.from_values(
                                   filename,
                                   start,
                                   1,
                                   line_number,
                                   len(file[line_number-1])))
                elif "ignore " in line:
                    yield (get_ignore_scope(line, "ignore "),
                           SourceRange.from_values(filename,
                                                   line_number,
                                                   1,
                                                   line_number+1,
                                                   len(file[line_number])))
        if stop_ignoring is False and start is not None:
            yield (bears,
                   SourceRange.from_values(filename,
                                           start,
                                           1,
                                           len(file),
                                           len(file[-1])))


def process_queues(processes,
                   control_queue,
                   local_result_dict,
                   global_result_dict,
                   file_dict,
                   print_results,
                   section,
                   log_printer):
    """
    Iterate the control queue and send the results recieved to the print_result
    method so that they can be presented to the user.

    :param processes:          List of processes which can be used to run
                               Bears.
    :param control_queue:      Containing control elements that indicate
                               whether there is a result available and which
                               bear it belongs to.
    :param local_result_dict:  Dictionary containing results respective to
                               local bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param global_result_dict: Dictionary containing results respective to
                               global bears. It is modified by the processes
                               i.e. results are added to it by multiple
                               processes.
    :param file_dict:          Dictionary containing file contents with
                               filename as keys.
    :param print_results:      Prints all given results appropriate to the
                               output medium.
    :return:                   Return True if all bears execute succesfully and
                               Results were delivered to the user. Else False.
    """
    file_diff_dict = {}
    retval = False
    # Number of processes working on local/global bears. They are count down
    # when the last queue element of that process is processed which may be
    # *after* the process has ended!
    local_processes = len(processes)
    global_processes = len(processes)
    global_result_buffer = []
    ignore_ranges = list(yield_ignore_ranges(file_dict))

    # One process is the logger thread
    while local_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
                local_processes -= 1
            elif control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED:
                global_processes -= 1
            elif control_elem == CONTROL_ELEMENT.LOCAL:
                assert local_processes != 0
                retval, res = print_result(local_result_dict[index],
                                           file_dict,
                                           retval,
                                           print_results,
                                           section,
                                           log_printer,
                                           file_diff_dict,
                                           ignore_ranges)
                local_result_dict[index] = res
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL
                global_result_buffer.append(index)
        except queue.Empty:
            if get_running_processes(processes) < 2:  # pragma: no cover
                # Recover silently, those branches are only
                # nondeterministically covered.
                break

    # Flush global result buffer
    for elem in global_result_buffer:
        retval, res = print_result(global_result_dict[elem],
                                   file_dict,
                                   retval,
                                   print_results,
                                   section,
                                   log_printer,
                                   file_diff_dict,
                                   ignore_ranges)
        global_result_dict[elem] = res

    # One process is the logger thread
    while global_processes > 1:
        try:
            control_elem, index = control_queue.get(timeout=0.1)

            if control_elem == CONTROL_ELEMENT.GLOBAL:
                retval, res = print_result(global_result_dict[index],
                                           file_dict,
                                           retval,
                                           print_results,
                                           section,
                                           log_printer,
                                           file_diff_dict,
                                           ignore_ranges)
                global_result_dict[index] = res
            else:
                assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
                global_processes -= 1
        except queue.Empty:
            if get_running_processes(processes) < 2:  # pragma: no cover
                # Recover silently, those branches are only
                # nondeterministically covered.
                break

    return retval


def simplify_section_result(section_result):
    """
    Takes in a section's result from ``execute_section`` and simplifies it
    for easy usage in other functions.

    :param section_result: The result of a section which was executed.
    :return:               Tuple containing:
                            - bool - True if results were yielded
                            - bool - True if unfixed results were yielded
                            - list - Results from all bears (local and global)
    """
    section_yielded_result = section_result[0]
    results_for_section = []
    for value in chain(section_result[1].values(),
                       section_result[2].values()):
        if value is None:
            continue

        for result in value:
            results_for_section.append(result)
    section_yielded_unfixed_results = len(results_for_section) > 0

    return (section_yielded_result,
            section_yielded_unfixed_results,
            results_for_section)


def execute_section(section,
                    global_bear_list,
                    local_bear_list,
                    print_results,
                    log_printer):
    """
    Executes the section with the given bears.

    The execute_section method does the following things:

    1. Prepare a Process
       -  Load files
       -  Create queues
    2. Spawn up one or more Processes
    3. Output results from the Processes
    4. Join all processes

    :param section:          The section to execute.
    :param global_bear_list: List of global bears belonging to the section.
    :param local_bear_list:  List of local bears belonging to the section.
    :param print_results:    Prints all given results appropriate to the
                             output medium.
    :param log_printer:      The log_printer to warn to.
    :return:                 Tuple containing a bool (True if results were
                             yielded, False otherwise), a Manager.dict
                             containing all local results(filenames are key)
                             and a Manager.dict containing all global bear
                             results (bear names are key) as well as the
                             file dictionary.
    """
    local_bear_list = Dependencies.resolve(local_bear_list)
    global_bear_list = Dependencies.resolve(global_bear_list)

    try:
        running_processes = int(section['jobs'])
    except ValueError:
        log_printer.warn("Unable to convert setting 'jobs' into a number. "
                         "Falling back to CPU count.")
        running_processes = get_cpu_count()
    except IndexError:
        running_processes = get_cpu_count()

    processes, arg_dict = instantiate_processes(section,
                                                local_bear_list,
                                                global_bear_list,
                                                running_processes,
                                                log_printer)

    logger_thread = LogPrinterThread(arg_dict["message_queue"],
                                     log_printer)
    # Start and join the logger thread along with the processes to run bears
    processes.append(logger_thread)

    for runner in processes:
        runner.start()

    try:
        return (process_queues(processes,
                               arg_dict["control_queue"],
                               arg_dict["local_result_dict"],
                               arg_dict["global_result_dict"],
                               arg_dict["file_dict"],
                               print_results,
                               section,
                               log_printer),
                arg_dict["local_result_dict"],
                arg_dict["global_result_dict"],
                arg_dict["file_dict"])
    finally:
        logger_thread.running = False

        for runner in processes:
            runner.join()


1			import multiprocessing
2			import os
3			import platform
4			import queue
5			import subprocess
6			from itertools import chain
7
8			from coalib.collecting import Dependencies
9			from coalib.collecting.Collectors import collect_files
10			from coalib.misc.StringConverter import StringConverter
11			from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
12			from coalib.processes.BearRunning import run
13			from coalib.processes.CONTROL_ELEMENT import CONTROL_ELEMENT
14			from coalib.processes.LogPrinterThread import LogPrinterThread
15			from coalib.results.Result import Result
16			from coalib.results.result_actions.ApplyPatchAction import ApplyPatchAction
17			from coalib.results.result_actions.PrintDebugMessageAction import (
18			PrintDebugMessageAction)
19			from coalib.results.result_actions.ShowPatchAction import ShowPatchAction
20			from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
21			from coalib.results.SourceRange import SourceRange
22			from coalib.settings.Setting import glob_list
23			from coalib.parsing.Globbing import fnmatch
24
25			ACTIONS = [ApplyPatchAction,
26			PrintDebugMessageAction,
27			ShowPatchAction]
28
29
30			def get_cpu_count():
31			try:
32			return multiprocessing.cpu_count()
33			# cpu_count is not implemented for some CPU architectures/OSes
34			except NotImplementedError: # pragma: no cover
35			return 2
36
37
38			def fill_queue(queue_fill, any_list):
39			"""
40			Takes element from a list and populates a queue with those elements.
41
42			:param queue_fill: The queue to be filled.
43			:param any_list: List containing the elements.
44			"""
45			for elem in any_list:
46			queue_fill.put(elem)
47
48
49			def get_running_processes(processes):
50			return sum((1 if process.is_alive() else 0) for process in processes)
51
52
53			def create_process_group(command_array, **kwargs):
54			if platform.system() == "Windows": # pragma: no cover
55			proc = subprocess.Popen(
56			command_array,
57			creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
58			**kwargs)
59			else:
60			proc = subprocess.Popen(command_array,
61			preexec_fn=os.setsid,
62			**kwargs)
63			return proc
64
65
66			def get_default_actions(section):
67			"""
68			Parses the key ``default_actions`` in the given section.
69
70			:param section: The section where to parse from.
71			:return: A dict with the bearname as keys and their default
72			actions as values and another dict that contains bears
73			and invalid action names.
74			"""
75			try:
76			default_actions = dict(section["default_actions"])
77			except IndexError:
78			return {}, {}
79
80			action_dict = {action.get_metadata().name: action for action in ACTIONS}
81			invalid_action_set = default_actions.values() - action_dict.keys()
82			invalid_actions = {}
83			if len(invalid_action_set) != 0:
84			invalid_actions = {
85			bear: action
86			for bear, action in default_actions.items()
87			if action in invalid_action_set}
88			for invalid in invalid_actions.keys():
89			del default_actions[invalid]
90
91			actions = {bearname: action_dict[action_name]
92			for bearname, action_name in default_actions.items()}
93			return actions, invalid_actions
94
95
96			def autoapply_actions(results,
97			file_dict,
98			file_diff_dict,
99			section,
100			log_printer):
101			"""
102			Auto-applies actions like defined in the given section.
103
104			:param results: A list of results.
105			:param file_dict: A dictionary containing the name of files and its
106			contents.
107			:param file_diff_dict: A dictionary that contains filenames as keys and
108			diff objects as values.
109			:param section: The section.
110			:param log_printer: A log printer instance to log messages on.
111			:return: A list of unprocessed results.
112			"""
113
114			default_actions, invalid_actions = get_default_actions(section)
115
116			for bearname, actionname in invalid_actions.items():
117			log_printer.warn("Selected default action {!r} for bear {!r} does "
118			"not exist. Ignoring action.".format(actionname,
119			bearname))
120
121			if len(default_actions) == 0:
122			# There's nothing to auto-apply.
123			return results
124
125			not_processed_results = []
126			for result in results:
127			try:
128			# Match full bear names deterministically, prioritized!
129			action = default_actions[result.origin]
130			except KeyError:
131			for bear_glob in default_actions:
132			if fnmatch(result.origin, bear_glob):
133			action = default_actions[bear_glob]
134			break
135			else:
136			not_processed_results.append(result)
137			continue
138
139			if not action.is_applicable(result, file_dict, file_diff_dict):
140			log_printer.warn("Selected default action {!r} for bear {!r} is "
141			"not applicable. Action not applied.".format(
142			action.get_metadata().name, result.origin))
143			not_processed_results.append(result)
144			continue
145
146			try:
147			action().apply_from_section(result,
148			file_dict,
149			file_diff_dict,
150			section)
151			log_printer.info("Applied {!r} on {} from {!r}.".format(
152			action.get_metadata().name,
153			result.location_repr(),
154			result.origin))
155			except Exception as ex:
156			not_processed_results.append(result)
157			log_printer.log_exception(
158			"Failed to execute action {!r} with error: {}.".format(
159			action.get_metadata().name, ex),
160			ex)
161			log_printer.debug("-> for result " + repr(result) + ".")
162
163			return not_processed_results
164
165
166			def check_result_ignore(result, ignore_ranges):
167			"""
168			Determines if the result has to be ignored.
169
170			:param result: The result that needs to be checked.
171			:param ignore_ranges: A list of tuples, each containing a list of lower
172			cased affected bearnames and a SourceRange to
173			ignore. If any of the bearname lists is empty, it
174			is considered an ignore range for all bears.
175			This may be a list of globbed bear wildcards.
176			:return: True if the result has to be ignored.
177			"""
178			for bears, range in ignore_ranges:
179			orig = result.origin.lower()
180			if (result.overlaps(range) and
181			(len(bears) == 0 or orig in bears or fnmatch(orig, bears))):
182			return True
183
184			return False
185
186
187			def print_result(results,
188			file_dict,
189			retval,
190			print_results,
191			section,
192			log_printer,
193			file_diff_dict,
194			ignore_ranges):
195			"""
196			Takes the results produced by each bear and gives them to the print_results
197			method to present to the user.
198
199			:param results: A list of results.
200			:param file_dict: A dictionary containing the name of files and its
201			contents.
202			:param retval: It is True if no results were yielded ever before.
203			If it is False this function will return False no
204			matter what happens. Else it depends on if this
205			invocation yields results.
206			:param print_results: A function that prints all given results appropriate
207			to the output medium.
208			:param file_diff_dict: A dictionary that contains filenames as keys and
209			diff objects as values.
210			:param ignore_ranges: A list of SourceRanges. Results that affect code in
211			any of those ranges will be ignored.
212			:return: Returns False if any results were yielded. Else
213			True.
214			"""
215			min_severity_str = str(section.get('min_severity', 'INFO')).upper()
216			min_severity = RESULT_SEVERITY.str_dict.get(min_severity_str, 'INFO')
217			results = list(filter(lambda result:
218			type(result) is Result and
219			result.severity >= min_severity and
220			not check_result_ignore(result, ignore_ranges),
221			results))
222
223			if bool(section.get('autoapply', 'true')):
224			patched_results = autoapply_actions(results,
225			file_dict,
226			file_diff_dict,
227			section,
228			log_printer)
229			else:
230			patched_results = results
231
232			print_results(log_printer,
233			section,
234			patched_results,
235			file_dict,
236			file_diff_dict)
237			return retval or len(results) > 0, patched_results
238
239
240			def get_file_dict(filename_list, log_printer):
241			"""
242			Reads all files into a dictionary.
243
244			:param filename_list: List of names of paths to files to get contents of.
245			:param log_printer: The logger which logs errors.
246			:return: Reads the content of each file into a dictionary
247			with filenames as keys.
248			"""
249			file_dict = {}
250			for filename in filename_list:
251			try:
252			with open(filename, "r", encoding="utf-8") as _file:
253			file_dict[filename] = tuple(_file.readlines())
254			except UnicodeDecodeError:
255			log_printer.warn("Failed to read file '{}'. It seems to contain "
256			"non-unicode characters. Leaving it "
257			"out.".format(filename))
258			except OSError as exception: # pragma: no cover
259			log_printer.log_exception("Failed to read file '{}' because of "
260			"an unknown error. Leaving it "
261			"out.".format(filename),
262			exception,
263			log_level=LOG_LEVEL.WARNING)
264
265			log_printer.debug("Files that will be checked:\n" +
266			"\n".join(file_dict.keys()))
267			return file_dict
268
269
270			def filter_raising_callables(it, exception, args, *kwargs):
271			"""
272			Filters all callable items inside the given iterator that raise the
273			given exceptions.
274
275			:param it: The iterator to filter.
276			:param exception: The (tuple of) exception(s) to filter for.
277			:param args: Positional arguments to pass to the callable.
278			:param kwargs: Keyword arguments to pass to the callable.
279			"""
280			for elem in it:
281			try:
282			yield elem(args, *kwargs)
283			except exception:
284			pass
285
286
287			def instantiate_bears(section,
288			local_bear_list,
289			global_bear_list,
290			file_dict,
291			message_queue):
292			"""
293			Instantiates each bear with the arguments it needs.
294
295			:param section: The section the bears belong to.
296			:param local_bear_list: List of local bear classes to instantiate.
297			:param global_bear_list: List of global bear classes to instantiate.
298			:param file_dict: Dictionary containing filenames and their
299			contents.
300			:param message_queue: Queue responsible to maintain the messages
301			delivered by the bears.
302			:return: The local and global bear instance lists.
303			"""
304			local_bear_list = [bear
305			for bear in filter_raising_callables(
306			local_bear_list,
307			RuntimeError,
308			section,
309			message_queue,
310			timeout=0.1)]
311
312			global_bear_list = [bear
313			for bear in filter_raising_callables(
314			global_bear_list,
315			RuntimeError,
316			file_dict,
317			section,
318			message_queue,
319			timeout=0.1)]
320
321			return local_bear_list, global_bear_list
322
323
324			def instantiate_processes(section,
325			local_bear_list,
326			global_bear_list,
327			job_count,
328			log_printer):
329			"""
330			Instantiate the number of processes that will run bears which will be
331			responsible for running bears in a multiprocessing environment.
332
333			:param section: The section the bears belong to.
334			:param local_bear_list: List of local bears belonging to the section.
335			:param global_bear_list: List of global bears belonging to the section.
336			:param job_count: Max number of processes to create.
337			:param log_printer: The log printer to warn to.
338			:return: A tuple containing a list of processes,
339			and the arguments passed to each process which are
340			the same for each object.
341			"""
342			filename_list = collect_files(
343			glob_list(section.get('files', "")),
344			log_printer,
345			ignored_file_paths=glob_list(section.get('ignore', "")),
346			limit_file_paths=glob_list(section.get('limit_files', "")))
347			file_dict = get_file_dict(filename_list, log_printer)
348
349			manager = multiprocessing.Manager()
350			global_bear_queue = multiprocessing.Queue()
351			filename_queue = multiprocessing.Queue()
352			local_result_dict = manager.dict()
353			global_result_dict = manager.dict()
354			message_queue = multiprocessing.Queue()
355			control_queue = multiprocessing.Queue()
356
357			bear_runner_args = {"file_name_queue": filename_queue,
358			"local_bear_list": local_bear_list,
359			"global_bear_list": global_bear_list,
360			"global_bear_queue": global_bear_queue,
361			"file_dict": file_dict,
362			"local_result_dict": local_result_dict,
363			"global_result_dict": global_result_dict,
364			"message_queue": message_queue,
365			"control_queue": control_queue,
366			"timeout": 0.1}
367
368			local_bear_list[:], global_bear_list[:] = instantiate_bears(
369			section,
370			local_bear_list,
371			global_bear_list,
372			file_dict,
373			message_queue)
374
375			fill_queue(filename_queue, file_dict.keys())
376			fill_queue(global_bear_queue, range(len(global_bear_list)))
377
378			return ([multiprocessing.Process(target=run, kwargs=bear_runner_args)
379			for i in range(job_count)],
380			bear_runner_args)
381
382
383			def get_ignore_scope(line, keyword):
384			"""
385			Retrieves the bears that are to be ignored defined in the given line.
386
387			:param line: The line containing the ignore declaration.
388			:param keyword: The keyword that was found. Everything after the rightmost
389			occurrence of it will be considered for the scope.
390			:return: A list of lower cased bearnames or an empty list (-> "all")
391			"""
392			toignore = line[line.rfind(keyword) + len(keyword):]
393			if toignore.startswith("all"):
394			return []
395			else:
396			return list(StringConverter(toignore, list_delimiters=', '))
397
398
399			def yield_ignore_ranges(file_dict):
400			"""
401			Yields tuples of affected bears and a SourceRange that shall be ignored for
402			those.
403
404			:param file_dict: The file dictionary.
405			"""
406			for filename, file in file_dict.items():
407			start = None
408			bears = []
409			stop_ignoring = False
410			for line_number, line in enumerate(file, start=1):
411			# Before lowering all lines ever read, first look for the biggest
412			# common substring, case sensitive: Ignore, start ignoring.
413			if 'gnor' in line:
414			line = line.lower()
415			if "start ignoring " in line:
416			start = line_number
417			bears = get_ignore_scope(line, "start ignoring ")
418			elif "stop ignoring" in line:
419			stop_ignoring = True
420			if start:
421			yield (bears,
422			SourceRange.from_values(
423			filename,
424			start,
425			1,
426			line_number,
427			len(file[line_number-1])))
428			elif "ignore " in line:
429			yield (get_ignore_scope(line, "ignore "),
430			SourceRange.from_values(filename,
431			line_number,
432			1,
433			line_number+1,
434			len(file[line_number])))
435			if stop_ignoring is False and start is not None:
436			yield (bears,
437			SourceRange.from_values(filename,
438			start,
439			1,
440			len(file),
441			len(file[-1])))
442
443
444			def process_queues(processes,
445			control_queue,
446			local_result_dict,
447			global_result_dict,
448			file_dict,
449			print_results,
450			section,
451			log_printer):
452			"""
453			Iterate the control queue and send the results recieved to the print_result
454			method so that they can be presented to the user.
455
456			:param processes: List of processes which can be used to run
457			Bears.
458			:param control_queue: Containing control elements that indicate
459			whether there is a result available and which
460			bear it belongs to.
461			:param local_result_dict: Dictionary containing results respective to
462			local bears. It is modified by the processes
463			i.e. results are added to it by multiple
464			processes.
465			:param global_result_dict: Dictionary containing results respective to
466			global bears. It is modified by the processes
467			i.e. results are added to it by multiple
468			processes.
469			:param file_dict: Dictionary containing file contents with
470			filename as keys.
471			:param print_results: Prints all given results appropriate to the
472			output medium.
473			:return: Return True if all bears execute succesfully and
474			Results were delivered to the user. Else False.
475			"""
476			file_diff_dict = {}
477			retval = False
478			# Number of processes working on local/global bears. They are count down
479			# when the last queue element of that process is processed which may be
480			# after the process has ended!
481			local_processes = len(processes)
482			global_processes = len(processes)
483			global_result_buffer = []
484			ignore_ranges = list(yield_ignore_ranges(file_dict))
485
486			# One process is the logger thread
487			while local_processes > 1:
488			try:
489			control_elem, index = control_queue.get(timeout=0.1)
490
491			if control_elem == CONTROL_ELEMENT.LOCAL_FINISHED:
492			local_processes -= 1
493			elif control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED:
494			global_processes -= 1
495			elif control_elem == CONTROL_ELEMENT.LOCAL:
496			assert local_processes != 0
497			retval, res = print_result(local_result_dict[index],
498			file_dict,
499			retval,
500			print_results,
501			section,
502			log_printer,
503			file_diff_dict,
504			ignore_ranges)
505			local_result_dict[index] = res
506			else:
507			assert control_elem == CONTROL_ELEMENT.GLOBAL
508			global_result_buffer.append(index)
509			except queue.Empty:
510			if get_running_processes(processes) < 2: # pragma: no cover
511			# Recover silently, those branches are only
512			# nondeterministically covered.
513			break
514
515			# Flush global result buffer
516			for elem in global_result_buffer:
517			retval, res = print_result(global_result_dict[elem],
518			file_dict,
519			retval,
520			print_results,
521			section,
522			log_printer,
523			file_diff_dict,
524			ignore_ranges)
525			global_result_dict[elem] = res
526
527			# One process is the logger thread
528			while global_processes > 1:
529			try:
530			control_elem, index = control_queue.get(timeout=0.1)
531
532			if control_elem == CONTROL_ELEMENT.GLOBAL:
533			retval, res = print_result(global_result_dict[index],
534			file_dict,
535			retval,
536			print_results,
537			section,
538			log_printer,
539			file_diff_dict,
540			ignore_ranges)
541			global_result_dict[index] = res
542			else:
543			assert control_elem == CONTROL_ELEMENT.GLOBAL_FINISHED
544			global_processes -= 1
545			except queue.Empty:
546			if get_running_processes(processes) < 2: # pragma: no cover
547			# Recover silently, those branches are only
548			# nondeterministically covered.
549			break
550
551			return retval
552
553
554			def simplify_section_result(section_result):
555			"""
556			Takes in a section's result from ``execute_section`` and simplifies it
557			for easy usage in other functions.
558
559			:param section_result: The result of a section which was executed.
560			:return: Tuple containing:
561			- bool - True if results were yielded
562			- bool - True if unfixed results were yielded
563			- list - Results from all bears (local and global)
564			"""
565			section_yielded_result = section_result[0]
566			results_for_section = []
567			for value in chain(section_result[1].values(),
568			section_result[2].values()):
569			if value is None:
570			continue
571
572			for result in value:
573			results_for_section.append(result)
574			section_yielded_unfixed_results = len(results_for_section) > 0
575
576			return (section_yielded_result,
577			section_yielded_unfixed_results,
578			results_for_section)
579
580
581			def execute_section(section,
582			global_bear_list,
583			local_bear_list,
584			print_results,
585			log_printer):
586			"""
587			Executes the section with the given bears.
588
589			The execute_section method does the following things:
590
591			1. Prepare a Process
592			- Load files
593			- Create queues
594			2. Spawn up one or more Processes
595			3. Output results from the Processes
596			4. Join all processes
597
598			:param section: The section to execute.
599			:param global_bear_list: List of global bears belonging to the section.
600			:param local_bear_list: List of local bears belonging to the section.
601			:param print_results: Prints all given results appropriate to the
602			output medium.
603			:param log_printer: The log_printer to warn to.
604			:return: Tuple containing a bool (True if results were
605			yielded, False otherwise), a Manager.dict
606			containing all local results(filenames are key)
607			and a Manager.dict containing all global bear
608			results (bear names are key) as well as the
609			file dictionary.
610			"""
611			local_bear_list = Dependencies.resolve(local_bear_list)
612			global_bear_list = Dependencies.resolve(global_bear_list)
613
614			try:
615			running_processes = int(section['jobs'])
616			except ValueError:
617			log_printer.warn("Unable to convert setting 'jobs' into a number. "
618			"Falling back to CPU count.")
619			running_processes = get_cpu_count()
620			except IndexError:
621			running_processes = get_cpu_count()
622
623			processes, arg_dict = instantiate_processes(section,
624			local_bear_list,
625			global_bear_list,
626			running_processes,
627			log_printer)
628
629			logger_thread = LogPrinterThread(arg_dict["message_queue"],
630			log_printer)
631			# Start and join the logger thread along with the processes to run bears
632			processes.append(logger_thread)
633
634			for runner in processes:
635			runner.start()
636
637			try:
638			return (process_queues(processes,
639			arg_dict["control_queue"],
640			arg_dict["local_result_dict"],
641			arg_dict["global_result_dict"],
642			arg_dict["file_dict"],
643			print_results,
644			section,
645			log_printer),
646			arg_dict["local_result_dict"],
647			arg_dict["global_result_dict"],
648			arg_dict["file_dict"])
649			finally:
650			logger_thread.running = False
651
652			for runner in processes:
653			runner.join()
654

coala-analyzer / coala

Pull Request — master (#2292)

autoapply_actions() D

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like