coalib.bearlib.abstractions.Lint._get_groupdict() - Code Metrics - Inspection of "Printers printers everywhere" - coala-analyzer/coala - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#1941)

by Abdeali

created 2016-03-22 06:56 UTC

coalib.bearlib.abstractions.Lint._get_groupdict() B

↳ Parent: coalib.bearlib.abstractions.Lint

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	4
dl	0
loc	23
rs	8.7972

import os
import re
import shutil
from subprocess import check_call, CalledProcessError, DEVNULL
import tempfile

from coalib.bears.Bear import Bear
from coalib.misc.Decorators import enforce_signature
from coalib.misc.Shell import escape_path_argument, run_shell_command
from coalib.results.Diff import Diff
from coalib.results.Result import Result
from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY


class Lint(Bear):
    """
    Deals with the creation of linting bears.

    For the tutorial see:
    http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html

    :param executable:                  The executable to run the linter.
    :param prerequisite_command:        The command to run as a prerequisite
                                        and is of type ``list``.
    :param prerequisites_fail_msg:      The message to be displayed if the
                                        prerequisite fails.
    :param arguments:                   The arguments to supply to the linter,
                                        such that the file name to be analyzed
                                        can be appended to the end. Note that
                                        we use ``.format()`` on the arguments -
                                        so, ``{abc}`` needs to be given as
                                        ``{{abc}}``. Currently, the following
                                        will be replaced:

                                         - ``{filename}`` - The filename passed
                                           to ``lint()``
                                         - ``{config_file}`` - The config file
                                           created using ``config_file()``

    :param output_regex:    The regex which will match the output of the linter
                            to get results. This is not used if
                            ``gives_corrected`` is set. This regex should give
                            out the following variables:

                             - line - The line where the issue starts.
                             - column - The column where the issue starts.
                             - end_line - The line where the issue ends.
                             - end_column - The column where the issue ends.
                             - severity - The severity of the issue.
                             - message - The message of the result.
                             - origin - The origin of the issue.

    :param diff_severity:   The severity to use for all results if
                            ``gives_corrected`` is set.
    :param diff_message:    The message to use for all results if
                            ``gives_corrected`` is set.
    :param use_stderr:      Uses stderr as the output stream is it's True.
    :param use_stdin:       Sends file as stdin instead of giving the file name.
    :param gives_corrected: True if the executable gives the corrected file
                            or just the issues.
    :param severity_map:    A dict where the keys are the possible severity
                            values the Linter gives out and the values are the
                            severity of the coala Result to set it to. If it is
                            not a dict, it is ignored.
    """
    executable = None
    prerequisite_command = None
    prerequisite_fail_msg = 'Unknown failure.'
    arguments = ""
    output_regex = re.compile(r'(?P<line>\d+)\.(?P<column>\d+)\|'
                              r'(?P<severity>\d+): (?P<message>.*)')
    diff_message = 'No result message was set'
    diff_severity = RESULT_SEVERITY.NORMAL
    use_stderr = False
    use_stdin = False
    gives_corrected = False
    severity_map = None

    def lint(self, filename=None, file=None):
        """
        Takes a file and lints it using the linter variables defined apriori.

        :param filename:  The name of the file to execute.
        :param file:      The contents of the file as a list of strings.
        """
        assert ((self.use_stdin and file is not None) or
                (not self.use_stdin and filename is not None))

        config_file = self.generate_config_file()
        self.command = self._create_command(filename=filename,
                                            config_file=config_file)

        stdin_input = "".join(file) if self.use_stdin else None
        stdout_output, stderr_output = run_shell_command(self.command,
                                                         stdin=stdin_input)
        self.stdout_output = tuple(stdout_output.splitlines(keepends=True))
        self.stderr_output = tuple(stderr_output.splitlines(keepends=True))
        results_output = (self.stderr_output if self.use_stderr
                          else self.stdout_output)
        results = self.process_output(results_output, filename, file)
        if not self.use_stderr:
            self._print_errors(self.stderr_output)

        if config_file:
            os.remove(config_file)

        return results

    def process_output(self, output, filename, file):
        """
        Take the output (from stdout or stderr) and use it to create Results.
        If the class variable ``gives_corrected`` is set to True, the
        ``_process_corrected()`` is called. If it is False,
        ``_process_issues()`` is called.

        :param output:   The output to be used to obtain Results from. The
                         output is either stdout or stderr depending on the
                         class variable ``use_stderr``.
        :param filename: The name of the file whose output is being processed.
        :param file:     The contents of the file whose output is being
                         processed.
        :return:         Generator which gives Results produced based on this
                         output.
        """
        if self.gives_corrected:
            return self._process_corrected(output, filename, file)
        else:
            return self._process_issues(output, filename)

    def _process_corrected(self, output, filename, file):
        """
        Process the output and use it to create Results by creating diffs.
        The diffs are created by comparing the output and the original file.

        :param output:   The corrected file contents.
        :param filename: The name of the file.
        :param file:     The original contents of the file.
        :return:         Generator which gives Results produced based on the
                         diffs created by comparing the original and corrected
                         contents.
        """
        for diff in self.__yield_diffs(file, output):
            yield Result(self,
                         self.diff_message,
                         affected_code=(diff.range(filename),),
                         diffs={filename: diff},
                         severity=self.diff_severity)

    def _process_issues(self, output, filename):
        """
        Process the output using the regex provided in ``output_regex`` and
        use it to create Results by using named captured groups from the regex.

        :param output:   The output to be parsed by regex.
        :param filename: The name of the file.
        :param file:     The original contents of the file.
        :return:         Generator which gives Results produced based on regex
                         matches using the ``output_regex`` provided and the
                         ``output`` parameter.
        """
        regex = self.output_regex
        if isinstance(regex, str):
            regex = regex % {"file_name": filename}

        # Note: We join ``output`` because the regex may want to capture
        #       multiple lines also.
        for match in re.finditer(regex, "".join(output)):
            yield self.match_to_result(match, filename)

    def _get_groupdict(self, match):
        """
        Convert a regex match's groups into a dictionary with data to be used
        to create a Result. This is used internally in ``match_to_result``.

        :param match:    The match got from regex parsing.
        :param filename: The name of the file from which this match is got.
        :return:         The dictionary containing the information:
                         - line - The line where the result starts.
                         - column - The column where the result starts.
                         - end_line - The line where the result ends.
                         - end_column - The column where the result ends.
                         - severity - The severity of the result.
                         - message - The message of the result.
                         - origin - The origin of the result.
        """
        groups = match.groupdict()
        if (
                isinstance(self.severity_map, dict) and
                "severity" in groups and
                groups["severity"] in self.severity_map):
            groups["severity"] = self.severity_map[groups["severity"]]
        return groups

    def _create_command(self, **kwargs):
        command = self.executable + ' ' + self.arguments
        for key in ("filename", "config_file"):
            kwargs[key] = escape_path_argument(kwargs.get(key, "") or "")
        return command.format(**kwargs)

    def _print_errors(self, errors):
        for line in filter(lambda error: bool(error.strip()), errors):
            self.warn(line)

    @staticmethod
    def __yield_diffs(file, new_file):
        if tuple(new_file) != tuple(file):
            wholediff = Diff.from_string_arrays(file, new_file)

            for diff in wholediff.split_diff():
                yield diff

    def match_to_result(self, match, filename):
        """
        Convert a regex match's groups into a coala Result object.

        :param match:    The match got from regex parsing.
        :param filename: The name of the file from which this match is got.
        :return:         The Result object.
        """
        groups = self._get_groupdict(match)

        # Pre process the groups
        for variable in ("line", "column", "end_line", "end_column"):
            if variable in groups and groups[variable]:
                groups[variable] = int(groups[variable])

        if "origin" in groups:
            groups['origin'] = "{} ({})".format(str(self.__class__.__name__),
                                                str(groups["origin"]))

        return Result.from_values(
            origin=groups.get("origin", self),
            message=groups.get("message", ""),
            file=filename,
            severity=int(groups.get("severity", RESULT_SEVERITY.NORMAL)),
            line=groups.get("line", None),
            column=groups.get("column", None),
            end_line=groups.get("end_line", None),
            end_column=groups.get("end_column", None))

    @classmethod
    def check_prerequisites(cls):
        """
        Checks for prerequisites required by the Linter Bear.

        It uses the class variables:
        -  ``executable`` - Checks that it is available in the PATH using
        ``shutil.which``.
        -  ``prerequisite_command`` - Checks that when this command is run,
        the exitcode is 0. If it is not zero, ``prerequisite_fail_msg``
        is gives as the failure message.

        If either of them is set to ``None`` that check is ignored.

        :return: True is all checks are valid, else False.
        """
        return cls._check_command(executable=cls.executable,
                                  command=cls.prerequisite_command,
                                  fail_msg=cls.prerequisite_fail_msg)

    @classmethod
    @enforce_signature
    def _check_command(cls, executable, command: (list, tuple, None), fail_msg):
        """
        Checks whether the required executable is found and the
        required command succesfully executes.

        The function is intended be used with classes having an
        executable, prerequisite_command and prerequisite_fail_msg.

        :param executable:   The executable to check for.
        :param command:      The command to check as a prerequisite.
        :param fail_msg:     The fail message to display when the
                             command doesn't return an exitcode of zero.

        :return: True if command successfully executes, or is not required.
                 not True otherwise, with a string containing a
                 detailed description of the error.
        """
        if cls._check_executable(executable):
            if command is None:
                return True  # when there are no prerequisites
            try:
                check_call(command, stdout=DEVNULL, stderr=DEVNULL)
                return True
            except (OSError, CalledProcessError):
                return fail_msg
        else:
            return repr(executable) + " is not installed."

    @staticmethod
    def _check_executable(executable):
        """
        Checks whether the needed executable is present in the system.

        :param executable: The executable to check for.

        :return: True if binary is present, or is not required.
                 not True otherwise, with a string containing a
                 detailed description of what's missing.
        """
        if executable is None:
            return True
        return shutil.which(executable) is not None

    def generate_config_file(self):
        """
        Generates a temporary config file.
        Note: The user of the function is responsible for deleting the
        tempfile when done with it.

        :return: The file name of the tempfile created.
        """
        config_lines = self.config_file()
        config_file = ""
        if config_lines is not None:
            for i, line in enumerate(config_lines):
                config_lines[i] = line if line.endswith("\n") else line + "\n"
            config_fd, config_file = tempfile.mkstemp()
            os.close(config_fd)
            with open(config_file, 'w') as conf_file:
                conf_file.writelines(config_lines)
        return config_file

    @staticmethod
    def config_file():
        """
        Returns a configuation file from the section given to the bear.
        The section is available in ``self.section``. To add the config
        file's name generated by this function to the arguments,
        use ``{config_file}``.

        :return: A list of lines of the config file to be used or None.
        """
        return None


1			import os
2			import re
3			import shutil
4			from subprocess import check_call, CalledProcessError, DEVNULL
5			import tempfile
6
7			from coalib.bears.Bear import Bear
8			from coalib.misc.Decorators import enforce_signature
9			from coalib.misc.Shell import escape_path_argument, run_shell_command
10			from coalib.results.Diff import Diff
11			from coalib.results.Result import Result
12			from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
13
14
15			class Lint(Bear):
16			"""
17			Deals with the creation of linting bears.
18
19			For the tutorial see:
20			http://coala.readthedocs.org/en/latest/Users/Tutorials/Linter_Bears.html
21
22			:param executable: The executable to run the linter.
23			:param prerequisite_command: The command to run as a prerequisite
24			and is of type ``list``.
25			:param prerequisites_fail_msg: The message to be displayed if the
26			prerequisite fails.
27			:param arguments: The arguments to supply to the linter,
28			such that the file name to be analyzed
29			can be appended to the end. Note that
30			we use ``.format()`` on the arguments -
31			so, ``{abc}`` needs to be given as
32			``{{abc}}``. Currently, the following
33			will be replaced:
34
35			- ``{filename}`` - The filename passed
36			to ``lint()``
37			- ``{config_file}`` - The config file
38			created using ``config_file()``
39
40			:param output_regex: The regex which will match the output of the linter
41			to get results. This is not used if
42			``gives_corrected`` is set. This regex should give
43			out the following variables:
44
45			- line - The line where the issue starts.
46			- column - The column where the issue starts.
47			- end_line - The line where the issue ends.
48			- end_column - The column where the issue ends.
49			- severity - The severity of the issue.
50			- message - The message of the result.
51			- origin - The origin of the issue.
52
53			:param diff_severity: The severity to use for all results if
54			``gives_corrected`` is set.
55			:param diff_message: The message to use for all results if
56			``gives_corrected`` is set.
57			:param use_stderr: Uses stderr as the output stream is it's True.
58			:param use_stdin: Sends file as stdin instead of giving the file name.
59			:param gives_corrected: True if the executable gives the corrected file
60			or just the issues.
61			:param severity_map: A dict where the keys are the possible severity
62			values the Linter gives out and the values are the
63			severity of the coala Result to set it to. If it is
64			not a dict, it is ignored.
65			"""
66			executable = None
67			prerequisite_command = None
68			prerequisite_fail_msg = 'Unknown failure.'
69			arguments = ""
70			output_regex = re.compile(r'(?P<line>\d+)\.(?P<column>\d+)\\|'
71			r'(?P<severity>\d+): (?P<message>.*)')
72			diff_message = 'No result message was set'
73			diff_severity = RESULT_SEVERITY.NORMAL
74			use_stderr = False
75			use_stdin = False
76			gives_corrected = False
77			severity_map = None
78
79			def lint(self, filename=None, file=None):
80			"""
81			Takes a file and lints it using the linter variables defined apriori.
82
83			:param filename: The name of the file to execute.
84			:param file: The contents of the file as a list of strings.
85			"""
86			assert ((self.use_stdin and file is not None) or
87			(not self.use_stdin and filename is not None))
88
89			config_file = self.generate_config_file()
90			self.command = self._create_command(filename=filename,
91			config_file=config_file)
92
93			stdin_input = "".join(file) if self.use_stdin else None
94			stdout_output, stderr_output = run_shell_command(self.command,
95			stdin=stdin_input)
96			self.stdout_output = tuple(stdout_output.splitlines(keepends=True))
97			self.stderr_output = tuple(stderr_output.splitlines(keepends=True))
98			results_output = (self.stderr_output if self.use_stderr
99			else self.stdout_output)
100			results = self.process_output(results_output, filename, file)
101			if not self.use_stderr:
102			self._print_errors(self.stderr_output)
103
104			if config_file:
105			os.remove(config_file)
106
107			return results
108
109			def process_output(self, output, filename, file):
110			"""
111			Take the output (from stdout or stderr) and use it to create Results.
112			If the class variable ``gives_corrected`` is set to True, the
113			``_process_corrected()`` is called. If it is False,
114			``_process_issues()`` is called.
115
116			:param output: The output to be used to obtain Results from. The
117			output is either stdout or stderr depending on the
118			class variable ``use_stderr``.
119			:param filename: The name of the file whose output is being processed.
120			:param file: The contents of the file whose output is being
121			processed.
122			:return: Generator which gives Results produced based on this
123			output.
124			"""
125			if self.gives_corrected:
126			return self._process_corrected(output, filename, file)
127			else:
128			return self._process_issues(output, filename)
129
130			def _process_corrected(self, output, filename, file):
131			"""
132			Process the output and use it to create Results by creating diffs.
133			The diffs are created by comparing the output and the original file.
134
135			:param output: The corrected file contents.
136			:param filename: The name of the file.
137			:param file: The original contents of the file.
138			:return: Generator which gives Results produced based on the
139			diffs created by comparing the original and corrected
140			contents.
141			"""
142			for diff in self.__yield_diffs(file, output):
143			yield Result(self,
144			self.diff_message,
145			affected_code=(diff.range(filename),),
146			diffs={filename: diff},
147			severity=self.diff_severity)
148
149			def _process_issues(self, output, filename):
150			"""
151			Process the output using the regex provided in ``output_regex`` and
152			use it to create Results by using named captured groups from the regex.
153
154			:param output: The output to be parsed by regex.
155			:param filename: The name of the file.
156			:param file: The original contents of the file.
157			:return: Generator which gives Results produced based on regex
158			matches using the ``output_regex`` provided and the
159			``output`` parameter.
160			"""
161			regex = self.output_regex
162			if isinstance(regex, str):
163			regex = regex % {"file_name": filename}
164
165			# Note: We join ``output`` because the regex may want to capture
166			# multiple lines also.
167			for match in re.finditer(regex, "".join(output)):
168			yield self.match_to_result(match, filename)
169
170			def _get_groupdict(self, match):
171			"""
172			Convert a regex match's groups into a dictionary with data to be used
173			to create a Result. This is used internally in ``match_to_result``.
174
175			:param match: The match got from regex parsing.
176			:param filename: The name of the file from which this match is got.
177			:return: The dictionary containing the information:
178			- line - The line where the result starts.
179			- column - The column where the result starts.
180			- end_line - The line where the result ends.
181			- end_column - The column where the result ends.
182			- severity - The severity of the result.
183			- message - The message of the result.
184			- origin - The origin of the result.
185			"""
186			groups = match.groupdict()
187			if (
188			isinstance(self.severity_map, dict) and
189			"severity" in groups and
190			groups["severity"] in self.severity_map):
191			groups["severity"] = self.severity_map[groups["severity"]]
192			return groups
193
194			def _create_command(self, **kwargs):
195			command = self.executable + ' ' + self.arguments
196			for key in ("filename", "config_file"):
197			kwargs[key] = escape_path_argument(kwargs.get(key, "") or "")
198			return command.format(**kwargs)
199
200			def _print_errors(self, errors):
201			for line in filter(lambda error: bool(error.strip()), errors):
202			self.warn(line)
203
204			@staticmethod
205			def __yield_diffs(file, new_file):
206			if tuple(new_file) != tuple(file):
207			wholediff = Diff.from_string_arrays(file, new_file)
208
209			for diff in wholediff.split_diff():
210			yield diff
211
212			def match_to_result(self, match, filename):
213			"""
214			Convert a regex match's groups into a coala Result object.
215
216			:param match: The match got from regex parsing.
217			:param filename: The name of the file from which this match is got.
218			:return: The Result object.
219			"""
220			groups = self._get_groupdict(match)
221
222			# Pre process the groups
223			for variable in ("line", "column", "end_line", "end_column"):
224			if variable in groups and groups[variable]:
225			groups[variable] = int(groups[variable])
226
227			if "origin" in groups:
228			groups['origin'] = "{} ({})".format(str(self.__class__.__name__),
229			str(groups["origin"]))
230
231			return Result.from_values(
232			origin=groups.get("origin", self),
233			message=groups.get("message", ""),
234			file=filename,
235			severity=int(groups.get("severity", RESULT_SEVERITY.NORMAL)),
236			line=groups.get("line", None),
237			column=groups.get("column", None),
238			end_line=groups.get("end_line", None),
239			end_column=groups.get("end_column", None))
240
241			@classmethod
242			def check_prerequisites(cls):
243			"""
244			Checks for prerequisites required by the Linter Bear.
245
246			It uses the class variables:
247			- ``executable`` - Checks that it is available in the PATH using
248			``shutil.which``.
249			- ``prerequisite_command`` - Checks that when this command is run,
250			the exitcode is 0. If it is not zero, ``prerequisite_fail_msg``
251			is gives as the failure message.
252
253			If either of them is set to ``None`` that check is ignored.
254
255			:return: True is all checks are valid, else False.
256			"""
257			return cls._check_command(executable=cls.executable,
258			command=cls.prerequisite_command,
259			fail_msg=cls.prerequisite_fail_msg)
260
261			@classmethod
262			@enforce_signature
263			def _check_command(cls, executable, command: (list, tuple, None), fail_msg):
264			"""
265			Checks whether the required executable is found and the
266			required command succesfully executes.
267
268			The function is intended be used with classes having an
269			executable, prerequisite_command and prerequisite_fail_msg.
270
271			:param executable: The executable to check for.
272			:param command: The command to check as a prerequisite.
273			:param fail_msg: The fail message to display when the
274			command doesn't return an exitcode of zero.
275
276			:return: True if command successfully executes, or is not required.
277			not True otherwise, with a string containing a
278			detailed description of the error.
279			"""
280			if cls._check_executable(executable):
281			if command is None:
282			return True # when there are no prerequisites
283			try:
284			check_call(command, stdout=DEVNULL, stderr=DEVNULL)
285			return True
286			except (OSError, CalledProcessError):
287			return fail_msg
288			else:
289			return repr(executable) + " is not installed."
290
291			@staticmethod
292			def _check_executable(executable):
293			"""
294			Checks whether the needed executable is present in the system.
295
296			:param executable: The executable to check for.
297
298			:return: True if binary is present, or is not required.
299			not True otherwise, with a string containing a
300			detailed description of what's missing.
301			"""
302			if executable is None:
303			return True
304			return shutil.which(executable) is not None
305
306			def generate_config_file(self):
307			"""
308			Generates a temporary config file.
309			Note: The user of the function is responsible for deleting the
310			tempfile when done with it.
311
312			:return: The file name of the tempfile created.
313			"""
314			config_lines = self.config_file()
315			config_file = ""
316			if config_lines is not None:
317			for i, line in enumerate(config_lines):
318			config_lines[i] = line if line.endswith("\n") else line + "\n"
319			config_fd, config_file = tempfile.mkstemp()
320			os.close(config_fd)
321			with open(config_file, 'w') as conf_file:
322			conf_file.writelines(config_lines)
323			return config_file
324
325			@staticmethod
326			def config_file():
327			"""
328			Returns a configuation file from the section given to the bear.
329			The section is available in ``self.section``. To add the config
330			file's name generated by this function to the arguments,
331			use ``{config_file}``.
332
333			:return: A list of lines of the config file to be used or None.
334			"""
335			return None
336

coala-analyzer / coala

Pull Request — master (#1941)

coalib.bearlib.abstractions.Lint._get_groupdict() B

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like