module_renamer.commands.track_modifications.walk_on_py_files() - Code Metrics - Inspection of "Add track command" - ESSS/module-renamer - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#2)

by William

created 2018-02-27 20:31 UTC

walk_on_py_files() A

↳ Parent: module_renamer.commands.track_modifications

Complexity

Conditions

Size

Total Lines	7
Code Lines	4

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	4
dl	0
loc	7
rs	9.4285
c	0
b	0
f	0
cc	3
nop	1

import ast
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
import fnmatch
import os
from collections import Counter, namedtuple
from contextlib import contextmanager

from click import ClickException, confirm, echo

from git import Repo

from tqdm import tqdm

from yapf.yapflib.yapf_api import FormatFile


CONFLICT_MSG = "\nUnfortunately, you moved two objects with the same name on " \
               "different paths.\nThis situation could be catastrophic while running " \
               "the script for rename the imports. \n" \
               "These are the imports that have conflict: \n -> {} \n"

INFORMATIVE_CONFLICT_MSG = 'Do you want to generate the file without this import? ' \
                           'Otherwise this script will be aborted.'
Import = namedtuple("Import", ["module", "name"])


@contextmanager
def branch_checkout(repo, branch_name):
    # type: (git.Repo, str) -> None
    """
    Change the current git branch for the branch informed on branch_name

    :param repo: A git.Repo object that represents the git repository from the project
    :type repo: git.Repo

    :param branch_name: Name of the branch that Git should move intoN
    :type branch_name: str
    """
    if (repo.is_dirty() or len(repo.untracked_files)) != 0:
        raise ClickException("The repository is dirty, please clean it first ")

    current_branch = repo.active_branch.name
    getattr(repo.heads, branch_name).checkout()

    try:
        yield
    finally:
        getattr(repo.heads, current_branch).checkout()


def track_modifications(**kwargs):
    """
    Command to track all modifications made between two different branches. The output will be a
    list written directly to a file (which will later be used by the script to rename the imports)

    :param kwargs:
    """
    project_path = os.path.join(kwargs['project_path'])
    repo = Repo(project_path)

    origin_branch = kwargs["origin_branch"]

    if kwargs['work_branch']:
        work_branch = kwargs['work_branch']
    else:
        work_branch = repo.active_branch.name

    if origin_branch == work_branch:
        raise ClickException(
            "Origin and Working branch are the same. "
            "Please, change your activate branch to where you made you changes on the code, "
            "or use the option --origin_branch and --work_branch  ."
        )
    file_counter = total_of_py_files_on_project(project_path)

    with branch_checkout(repo, origin_branch):
        origin_import_list = {imp for imp in get_imports(project_path, file_counter)}

    with branch_checkout(repo, work_branch):
        working_import_list = {imp for imp in get_imports(project_path, file_counter)}

    modified_imports = generate_list_with_modified_imports(origin_import_list, working_import_list)
    write_list_to_file(modified_imports, kwargs['output_file'])


def write_list_to_file(list_with_modified_imports, file_name):
    """
    Write the list of modified imports on a python file, the python file per default will be named
    "list_output.py" but can be changed by passing the argument --output_file

    The name of the list (inside the file) cannot be changed since it will be used later on the
    script for renaming the project.
    """
    echo('Generating the file {0}'.format(file_name))

    with open(file_name, 'w') as file:
        file.write("imports_to_move = ")
        file.writelines(repr(list(list_with_modified_imports)))

    FormatFile(file_name, in_place=True)


def generate_list_with_modified_imports(origin_import_list, working_import_list):

    """
    This methods looks for imports that keep the same name but has has different modules path

    :return: A list with unique elements that has the same name but different modules path
    :rtype: set
    """

    origin_filtered, working_filtered = _filter_import(origin_import_list, working_import_list)

    moved_imports = _find_moved_imports(origin_filtered, working_filtered)

    list_with_modified_imports = _check_for_conflicts(moved_imports)

    return list_with_modified_imports


def _filter_import(origin_import_list, working_import_list):
    """
    Create a new set with elements present origin_list or working_list but not on both,
    this helps to filter classes that could have same name but different modules.
    """
    difference = origin_import_list.symmetric_difference(working_import_list)
    origin_filtered = origin_import_list.intersection(difference)
    working_filtered = working_import_list.intersection(difference)
    return origin_filtered, working_filtered


def _find_moved_imports(origin_filtered, working_filtered):
    list_with_modified_imports = {
        (origin.module + "." + origin.name, working.module + "." + working.name)
        for origin in origin_filtered
        for working in working_filtered
        if working.name is not '*'

        if origin.name == working.name
        if origin.module != working.module
    }
    return list_with_modified_imports


def _check_for_conflicts(list_with_modified_imports):
    import_from = [modified_import[0] for modified_import in list(list_with_modified_imports)]
    imports_with_conflict = [class_name
                             for class_name, number_of_occurrences in Counter(import_from).items()
                             if number_of_occurrences > 1]
    if len(imports_with_conflict) > 0:

        echo(CONFLICT_MSG.format('\n -> '.join(imports_with_conflict)))

        if confirm(INFORMATIVE_CONFLICT_MSG, abort=True):
            list_with_modified_imports = [modified_import for modified_import in
                                          list_with_modified_imports
                                          if modified_import[0] not in imports_with_conflict
                                          if modified_import[1] not in imports_with_conflict]
    return list_with_modified_imports


def get_imports(project_path, file_counter):
    # type: (str) -> Import
    """
    Look for all .py files on the given project path and return the import statements found on
    each file.

    Note.: I inserted the TQDM here because was the only way that I could have an accurate
    progress bar, feel free to share any thoughts or tips on how to improve this progress bar =)

    :type project_path: str
    :rtype: commands.utils.Import
    """
    with tqdm(total=file_counter, unit='files', leave=False, desc=project_path) as pbar:
        for file_path in walk_on_py_files(project_path):
            pbar.update()
            with open(file_path, mode='r') as file:
                file_content = ast.parse(file.read(), file_path)

            for node in ast.iter_child_nodes(file_content):
                if isinstance(node, ast.Import):
                    module = ''
                elif isinstance(node, ast.ImportFrom):
                    # node.module can be None on the following situation
                    #   from . import foo
                    if node.module is not None:
                        module = node.module
                    else:
                        module = ''
                else:
                    continue

                for n in node.names:

                    yield Import(module, n.name)


def walk_on_py_files(folder):
    """
    Walk through each python files in a directory
    """
    for dir_path, _, files in os.walk(folder):
        for filename in fnmatch.filter(files, '*.py'):
            yield os.path.abspath(os.path.join(dir_path, filename))


def total_of_py_files_on_project(project_path):
    """
    Helper function to find the total number of py files that needs to be iterated.
    :return: Total number of py files
    """
    file_counter = 0
    for _ in walk_on_py_files(project_path):
        file_counter += 1
    return file_counter


1			import ast
			1 ignored issue – show Coding Style introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report This module should have a docstring. The coding style of this project requires that you add a docstring to this code element. Below, you find an example for methods: class SomeClass: def some_method(self): """Do x and return foo.""" If you would like to know more about docstrings, we recommend to read PEP-257: Docstring Conventions. Loading history...
2			import fnmatch
3			import os
4			from collections import Counter, namedtuple
5			from contextlib import contextmanager
6
7			from click import ClickException, confirm, echo
			0 ignored issues – show introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report Unable to import 'click' Loading history...
8			from git import Repo
			0 ignored issues – show introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report Unable to import 'git' Loading history...
9			from tqdm import tqdm
			0 ignored issues – show introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report Unable to import 'tqdm' Loading history...
10			from yapf.yapflib.yapf_api import FormatFile
			0 ignored issues – show introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report Unable to import 'yapf.yapflib.yapf_api' Loading history...
11
12			CONFLICT_MSG = "\nUnfortunately, you moved two objects with the same name on " \
13			"different paths.\nThis situation could be catastrophic while running " \
14			"the script for rename the imports. \n" \
15			"These are the imports that have conflict: \n -> {} \n"
16
17			INFORMATIVE_CONFLICT_MSG = 'Do you want to generate the file without this import? ' \
18			'Otherwise this script will be aborted.'
19			Import = namedtuple("Import", ["module", "name"])
20
21
22			@contextmanager
23			def branch_checkout(repo, branch_name):
24			# type: (git.Repo, str) -> None
25			"""
26			Change the current git branch for the branch informed on branch_name
27
28			:param repo: A git.Repo object that represents the git repository from the project
29			:type repo: git.Repo
30
31			:param branch_name: Name of the branch that Git should move intoN
32			:type branch_name: str
33			"""
34			if (repo.is_dirty() or len(repo.untracked_files)) != 0:
35			raise ClickException("The repository is dirty, please clean it first ")
36
37			current_branch = repo.active_branch.name
38			getattr(repo.heads, branch_name).checkout()
39
40			try:
41			yield
42			finally:
43			getattr(repo.heads, current_branch).checkout()
44
45
46			def track_modifications(**kwargs):
47			"""
48			Command to track all modifications made between two different branches. The output will be a
49			list written directly to a file (which will later be used by the script to rename the imports)
50
51			:param kwargs:
52			"""
53			project_path = os.path.join(kwargs['project_path'])
54			repo = Repo(project_path)
55
56			origin_branch = kwargs["origin_branch"]
57
58			if kwargs['work_branch']:
59			work_branch = kwargs['work_branch']
60			else:
61			work_branch = repo.active_branch.name
62
63			if origin_branch == work_branch:
64			raise ClickException(
65			"Origin and Working branch are the same. "
66			"Please, change your activate branch to where you made you changes on the code, "
67			"or use the option --origin_branch and --work_branch ."
68			)
69			file_counter = total_of_py_files_on_project(project_path)
70
71			with branch_checkout(repo, origin_branch):
72			origin_import_list = {imp for imp in get_imports(project_path, file_counter)}
73
74			with branch_checkout(repo, work_branch):
75			working_import_list = {imp for imp in get_imports(project_path, file_counter)}
76
77			modified_imports = generate_list_with_modified_imports(origin_import_list, working_import_list)
78			write_list_to_file(modified_imports, kwargs['output_file'])
79
80
81			def write_list_to_file(list_with_modified_imports, file_name):
82			"""
83			Write the list of modified imports on a python file, the python file per default will be named
84			"list_output.py" but can be changed by passing the argument --output_file
85
86			The name of the list (inside the file) cannot be changed since it will be used later on the
87			script for renaming the project.
88			"""
89			echo('Generating the file {0}'.format(file_name))
90
91			with open(file_name, 'w') as file:
92			file.write("imports_to_move = ")
93			file.writelines(repr(list(list_with_modified_imports)))
94
95			FormatFile(file_name, in_place=True)
96
97
98			def generate_list_with_modified_imports(origin_import_list, working_import_list):
			1 ignored issue – show Coding Style Naming introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report The name `generate_list_with_modified_imports` does not conform to the function naming conventions (`(([a-z][a-z0-9_]{2,30})\|(_[a-z0-9_]*))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
99			"""
100			This methods looks for imports that keep the same name but has has different modules path
101
102			:return: A list with unique elements that has the same name but different modules path
103			:rtype: set
104			"""
105
106			origin_filtered, working_filtered = _filter_import(origin_import_list, working_import_list)
107
108			moved_imports = _find_moved_imports(origin_filtered, working_filtered)
109
110			list_with_modified_imports = _check_for_conflicts(moved_imports)
111
112			return list_with_modified_imports
113
114
115			def _filter_import(origin_import_list, working_import_list):
116			"""
117			Create a new set with elements present origin_list or working_list but not on both,
118			this helps to filter classes that could have same name but different modules.
119			"""
120			difference = origin_import_list.symmetric_difference(working_import_list)
121			origin_filtered = origin_import_list.intersection(difference)
122			working_filtered = working_import_list.intersection(difference)
123			return origin_filtered, working_filtered
124
125
126			def _find_moved_imports(origin_filtered, working_filtered):
127			list_with_modified_imports = {
128			(origin.module + "." + origin.name, working.module + "." + working.name)
129			for origin in origin_filtered
130			for working in working_filtered
131			if working.name is not '*'
			0 ignored issues – show introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report Comparison to literal Loading history...
132			if origin.name == working.name
133			if origin.module != working.module
134			}
135			return list_with_modified_imports
136
137
138			def _check_for_conflicts(list_with_modified_imports):
139			import_from = [modified_import[0] for modified_import in list(list_with_modified_imports)]
140			imports_with_conflict = [class_name
141			for class_name, number_of_occurrences in Counter(import_from).items()
142			if number_of_occurrences > 1]
143			if len(imports_with_conflict) > 0:
			0 ignored issues – show Unused Code introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report Do not use `len(SEQUENCE)` as condition value Loading history...
144			echo(CONFLICT_MSG.format('\n -> '.join(imports_with_conflict)))
145
146			if confirm(INFORMATIVE_CONFLICT_MSG, abort=True):
147			list_with_modified_imports = [modified_import for modified_import in
148			list_with_modified_imports
149			if modified_import[0] not in imports_with_conflict
150			if modified_import[1] not in imports_with_conflict]
151			return list_with_modified_imports
152
153
154			def get_imports(project_path, file_counter):
155			# type: (str) -> Import
156			"""
157			Look for all .py files on the given project path and return the import statements found on
158			each file.
159
160			Note.: I inserted the TQDM here because was the only way that I could have an accurate
161			progress bar, feel free to share any thoughts or tips on how to improve this progress bar =)
162
163			:type project_path: str
164			:rtype: commands.utils.Import
165			"""
166			with tqdm(total=file_counter, unit='files', leave=False, desc=project_path) as pbar:
167			for file_path in walk_on_py_files(project_path):
168			pbar.update()
169			with open(file_path, mode='r') as file:
170			file_content = ast.parse(file.read(), file_path)
171
172			for node in ast.iter_child_nodes(file_content):
173			if isinstance(node, ast.Import):
174			module = ''
175			elif isinstance(node, ast.ImportFrom):
176			# node.module can be None on the following situation
177			# from . import foo
178			if node.module is not None:
179			module = node.module
180			else:
181			module = ''
182			else:
183			continue
184
185			for n in node.names:
			0 ignored issues – show Coding Style Naming introduced 2018-02-27 20:32 UTC by Report Bug Copy Issue Report The name `n` does not conform to the variable naming conventions (`(([a-z][a-z0-9_]{2,30})\|(_[a-z0-9_]*))$`). This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
186			yield Import(module, n.name)
187
188
189			def walk_on_py_files(folder):
190			"""
191			Walk through each python files in a directory
192			"""
193			for dir_path, _, files in os.walk(folder):
194			for filename in fnmatch.filter(files, '*.py'):
195			yield os.path.abspath(os.path.join(dir_path, filename))
196
197
198			def total_of_py_files_on_project(project_path):
199			"""
200			Helper function to find the total number of py files that needs to be iterated.
201			:return: Total number of py files
202			"""
203			file_counter = 0
204			for _ in walk_on_py_files(project_path):
205			file_counter += 1
206			return file_counter
207

ESSS / module-renamer

Pull Request — master (#2)

walk_on_py_files() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like