DownloadGitRepoAction - Code Metrics - Inspection of "Add "CACHE" suffix to the module variable name to..." - Plexxi/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — develop ( 6e30e7...c4b627 )

by Plexxi

created 2016-11-04 10:00 UTC

DownloadGitRepoAction B

↳ Parent: Project

Complexity

Total Complexity

Size/Duplication

Total Lines	251
Duplicated Lines	0 %

Importance

Changes

Metric	Value
dl	0
loc	251
rs	7.4757
c	0
b	0
f	0
wmc	53

How to fix Complexity

# -*- coding: utf-8 -*-
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
import hashlib
import stat
import re

import six
import yaml
from git.repo import Repo
from gitdb.exc import BadName, BadObject
from lockfile import LockFile

from st2common.runners.base_action import Action
from st2common.content import utils
from st2common.services.packs import get_pack_from_index
from st2common.util.green import shell

MANIFEST_FILE = 'pack.yaml'
CONFIG_FILE = 'config.yaml'
GITINFO_FILE = '.gitinfo'
PACK_RESERVE_CHARACTER = '.'
PACK_VERSION_SEPARATOR = '='
SEMVER_REGEX = (r"^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)"
                r"(?:-[\da-z\-]+(?:\.[\da-z\-]+)*)?(?:\+[\da-z\-]+(?:\.[\da-z\-]+)*)?$")


class DownloadGitRepoAction(Action):
    def __init__(self, config=None, action_service=None):
        super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)

    def run(self, packs, abs_repo_base, verifyssl=True):
        result = {}

        for pack in packs:
            pack_url, pack_version = self._get_repo_url(pack)
            temp_dir = hashlib.md5(pack_url).hexdigest()

            with LockFile('/tmp/%s' % (temp_dir)):
                try:
                    user_home = os.path.expanduser('~')
                    abs_local_path = os.path.join(user_home, temp_dir)
                    self._clone_repo(temp_dir=abs_local_path, repo_url=pack_url,
                                     verifyssl=verifyssl, ref=pack_version)
                    pack_name = self._get_pack_name(abs_local_path)
                    result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path)
                finally:
                    self._cleanup_repo(abs_local_path)

        return self._validate_result(result=result, repo_url=pack_url)

    @staticmethod
    def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
        # Switch to non-interactive mode
        os.environ['GIT_TERMINAL_PROMPT'] = '0'

        # Disable SSL cert checking if explictly asked
        if not verifyssl:
            os.environ['GIT_SSL_NO_VERIFY'] = 'true'

        # Clone the repo from git; we don't use shallow copying
        # because we want the user to work with the repo in the
        # future.
        repo = Repo.clone_from(repo_url, temp_dir)

        # Try to match the reference to a commit hash, a tag, or "master"
        gitref = DownloadGitRepoAction._get_gitref(repo, ref)

        # Try to match the reference to a "vX.Y.Z" tag
        if not gitref and re.match(SEMVER_REGEX, ref):
            gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)

        # Try to match the reference to a branch name
        if not gitref:
            gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)

        # Giving up ¯\_(ツ)_/¯
        if not gitref:
            raise ValueError(
                "\"%s\" is not a valid version, hash, tag, or branch in %s." % (ref, repo_url)
            )

        # We're trying to figure out which branch the ref is actually on,
        # since there's no direct way to check for this in git-python.
        branches = repo.git.branch('--color=never', '--all', '--contains', gitref.hexsha)
        branches = branches.replace('*', '').split()
        if 'master' not in branches:
            branch = branches[0]
            repo.git.checkout('--track', branches[0])
            branch = repo.head.reference
        else:
            branch = 'master'

        repo.git.checkout('-B', branch, gitref.hexsha)

        return temp_dir

    def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
        desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
        if desired:
            to = abs_repo_base
            dest_pack_path = os.path.join(abs_repo_base, pack_name)
            if os.path.exists(dest_pack_path):
                self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
                                  dest_pack_path)

                # Ensure to preserve any existing configuration
                old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
                new_config_file = os.path.join(abs_local_path, CONFIG_FILE)

                if os.path.isfile(old_config_file):
                    shutil.move(old_config_file, new_config_file)

                shutil.rmtree(dest_pack_path)

            self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
            shutil.move(abs_local_path, dest_pack_path)
            # post move fix all permissions.
            self._apply_pack_permissions(pack_path=dest_pack_path)
            message = 'Success.'
        elif message:
            message = 'Failure : %s' % message

        return (desired, message)

    def _apply_pack_permissions(self, pack_path):
        """
        Will recursively apply permission 770 to pack and its contents.
        """
        # 1. switch owner group to configured group
        pack_group = utils.get_pack_group()
        if pack_group:
            shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        # 2. Setup the right permissions and group ownership
        # These mask is same as mode = 775
        mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        os.chmod(pack_path, mode)

        # Yuck! Since os.chmod does not support chmod -R walk manually.
        for root, dirs, files in os.walk(pack_path):
            for d in dirs:
                os.chmod(os.path.join(root, d), mode)
            for f in files:
                os.chmod(os.path.join(root, f), mode)

    @staticmethod
    def _is_desired_pack(abs_pack_path, pack_name):
        # path has to exist.
        if not os.path.exists(abs_pack_path):
            return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
                    (pack_name))
        # should not include reserve characters
        if PACK_RESERVE_CHARACTER in pack_name:
            return (False, 'Pack name "%s" contains reserve character "%s"' %
                    (pack_name, PACK_RESERVE_CHARACTER))
        # must contain a manifest file. Empty file is ok for now.
        if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
            return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
        return (True, '')

    @staticmethod
    def _cleanup_repo(abs_local_path):
        # basic lock checking etc?
        if os.path.isdir(abs_local_path):
            shutil.rmtree(abs_local_path)

    @staticmethod
    def _validate_result(result, repo_url):
        atleast_one_success = False
        sanitized_result = {}
        for k, v in six.iteritems(result):
            atleast_one_success |= v[0]
            sanitized_result[k] = v[1]

        if not atleast_one_success:
            message_list = []
            message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
            message_list.append('Errors:')

            for pack, value in result.items():
                success, error = value
                message_list.append(error)

            message = '\n'.join(message_list)
            raise Exception(message)

        return sanitized_result

    @staticmethod
    def _get_repo_url(pack):
        pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
        name_or_url = pack_and_version[0]
        version = pack_and_version[1] if len(pack_and_version) > 1 else None

        if len(name_or_url.split('/')) == 1:
            pack = get_pack_from_index(name_or_url)
            if not pack:
                raise Exception('No record of the "%s" pack in the index.' % name_or_url)
            return (pack['repo_url'], version)
        else:
            return (DownloadGitRepoAction._eval_repo_url(name_or_url), version)

    @staticmethod
    def _eval_repo_url(repo_url):
        """Allow passing short GitHub style URLs"""
        if not repo_url:
            raise Exception('No valid repo_url provided or could be inferred.')
        has_git_extension = repo_url.endswith('.git')
        if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
            url = "https://github.com/{}".format(repo_url)
        else:
            url = repo_url
        return url if has_git_extension else "{}.git".format(url)

    @staticmethod
    def _get_pack_name(pack_dir):
        """
        Read pack name from the metadata file and sanitize it.
        """
        with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file:
            pack_meta = yaml.load(manifest_file)
        return pack_meta['name'].replace(' ', '-').lower()

    @staticmethod
    def _get_gitref(repo, ref):
        try:
            return repo.commit(ref)
        except (BadName, BadObject):
            return False


GitHub Access Token became invalid

Push — develop ( 6e30e7...c4b627 )

DownloadGitRepoAction B

Complexity

Size/Duplication

Importance

How to fix Complexity

Complex Class

1			# -- coding: utf-8 --
2			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
3			# contributor license agreements. See the NOTICE file distributed with
4			# this work for additional information regarding copyright ownership.
5			# The ASF licenses this file to You under the Apache License, Version 2.0
6			# (the "License"); you may not use this file except in compliance with
7			# the License. You may obtain a copy of the License at
8			#
9			# http://www.apache.org/licenses/LICENSE-2.0
10			#
11			# Unless required by applicable law or agreed to in writing, software
12			# distributed under the License is distributed on an "AS IS" BASIS,
13			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14			# See the License for the specific language governing permissions and
15			# limitations under the License.
16
17			import os
18			import shutil
19			import hashlib
20			import stat
21			import re
22
23			import six
24			import yaml
25			from git.repo import Repo
26			from gitdb.exc import BadName, BadObject
27			from lockfile import LockFile
28
29			from st2common.runners.base_action import Action
30			from st2common.content import utils
31			from st2common.services.packs import get_pack_from_index
32			from st2common.util.green import shell
33
34			MANIFEST_FILE = 'pack.yaml'
35			CONFIG_FILE = 'config.yaml'
36			GITINFO_FILE = '.gitinfo'
37			PACK_RESERVE_CHARACTER = '.'
38			PACK_VERSION_SEPARATOR = '='
39			SEMVER_REGEX = (r"^(?:0\|[1-9]\d)\.(?:0\|[1-9]\d)\.(?:0\|[1-9]\d*)"
40			r"(?:-[\da-z\-]+(?:\.[\da-z\-]+))?(?:\+[\da-z\-]+(?:\.[\da-z\-]+))?$")
41
42
43			class DownloadGitRepoAction(Action):
44			def __init__(self, config=None, action_service=None):
45			super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)
46
47			def run(self, packs, abs_repo_base, verifyssl=True):
48			result = {}
49
50			for pack in packs:
51			pack_url, pack_version = self._get_repo_url(pack)
52			temp_dir = hashlib.md5(pack_url).hexdigest()
53
54			with LockFile('/tmp/%s' % (temp_dir)):
55			try:
56			user_home = os.path.expanduser('~')
57			abs_local_path = os.path.join(user_home, temp_dir)
58			self._clone_repo(temp_dir=abs_local_path, repo_url=pack_url,
59			verifyssl=verifyssl, ref=pack_version)
60			pack_name = self._get_pack_name(abs_local_path)
61			result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path)
62			finally:
63			self._cleanup_repo(abs_local_path)
64
65			return self._validate_result(result=result, repo_url=pack_url)
66
67			@staticmethod
68			def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
69			# Switch to non-interactive mode
70			os.environ['GIT_TERMINAL_PROMPT'] = '0'
71
72			# Disable SSL cert checking if explictly asked
73			if not verifyssl:
74			os.environ['GIT_SSL_NO_VERIFY'] = 'true'
75
76			# Clone the repo from git; we don't use shallow copying
77			# because we want the user to work with the repo in the
78			# future.
79			repo = Repo.clone_from(repo_url, temp_dir)
80
81			# Try to match the reference to a commit hash, a tag, or "master"
82			gitref = DownloadGitRepoAction._get_gitref(repo, ref)
83
84			# Try to match the reference to a "vX.Y.Z" tag
85			if not gitref and re.match(SEMVER_REGEX, ref):
86			gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
87
88			# Try to match the reference to a branch name
89			if not gitref:
90			gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
91
92			# Giving up ¯\_(ツ)_/¯
93			if not gitref:
94			raise ValueError(
95			"\"%s\" is not a valid version, hash, tag, or branch in %s." % (ref, repo_url)
96			)
97
98			# We're trying to figure out which branch the ref is actually on,
99			# since there's no direct way to check for this in git-python.
100			branches = repo.git.branch('--color=never', '--all', '--contains', gitref.hexsha)
101			branches = branches.replace('*', '').split()
102			if 'master' not in branches:
103			branch = branches[0]
104			repo.git.checkout('--track', branches[0])
105			branch = repo.head.reference
106			else:
107			branch = 'master'
108
109			repo.git.checkout('-B', branch, gitref.hexsha)
110
111			return temp_dir
112
113			def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
114			desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
115			if desired:
116			to = abs_repo_base
117			dest_pack_path = os.path.join(abs_repo_base, pack_name)
118			if os.path.exists(dest_pack_path):
119			self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
120			dest_pack_path)
121
122			# Ensure to preserve any existing configuration
123			old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
124			new_config_file = os.path.join(abs_local_path, CONFIG_FILE)
125
126			if os.path.isfile(old_config_file):
127			shutil.move(old_config_file, new_config_file)
128
129			shutil.rmtree(dest_pack_path)
130
131			self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
132			shutil.move(abs_local_path, dest_pack_path)
133			# post move fix all permissions.
134			self._apply_pack_permissions(pack_path=dest_pack_path)
135			message = 'Success.'
136			elif message:
137			message = 'Failure : %s' % message
138
139			return (desired, message)
140
141			def _apply_pack_permissions(self, pack_path):
142			"""
143			Will recursively apply permission 770 to pack and its contents.
144			"""
145			# 1. switch owner group to configured group
146			pack_group = utils.get_pack_group()
147			if pack_group:
148			shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
149
150			# 2. Setup the right permissions and group ownership
151			# These mask is same as mode = 775
152			mode = stat.S_IRWXU \| stat.S_IRWXG \| stat.S_IROTH \| stat.S_IXOTH
153			os.chmod(pack_path, mode)
154
155			# Yuck! Since os.chmod does not support chmod -R walk manually.
156			for root, dirs, files in os.walk(pack_path):
157			for d in dirs:
158			os.chmod(os.path.join(root, d), mode)
159			for f in files:
160			os.chmod(os.path.join(root, f), mode)
161
162			@staticmethod
163			def _is_desired_pack(abs_pack_path, pack_name):
164			# path has to exist.
165			if not os.path.exists(abs_pack_path):
166			return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
167			(pack_name))
168			# should not include reserve characters
169			if PACK_RESERVE_CHARACTER in pack_name:
170			return (False, 'Pack name "%s" contains reserve character "%s"' %
171			(pack_name, PACK_RESERVE_CHARACTER))
172			# must contain a manifest file. Empty file is ok for now.
173			if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
174			return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
175			return (True, '')
176
177			@staticmethod
178			def _cleanup_repo(abs_local_path):
179			# basic lock checking etc?
180			if os.path.isdir(abs_local_path):
181			shutil.rmtree(abs_local_path)
182
183			@staticmethod
184			def _validate_result(result, repo_url):
185			atleast_one_success = False
186			sanitized_result = {}
187			for k, v in six.iteritems(result):
188			atleast_one_success \|= v[0]
189			sanitized_result[k] = v[1]
190
191			if not atleast_one_success:
192			message_list = []
193			message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
194			message_list.append('Errors:')
195
196			for pack, value in result.items():
197			success, error = value
198			message_list.append(error)
199
200			message = '\n'.join(message_list)
201			raise Exception(message)
202
203			return sanitized_result
204
205			@staticmethod
206			def _get_repo_url(pack):
207			pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
208			name_or_url = pack_and_version[0]
209			version = pack_and_version[1] if len(pack_and_version) > 1 else None
210
211			if len(name_or_url.split('/')) == 1:
212			pack = get_pack_from_index(name_or_url)
213			if not pack:
214			raise Exception('No record of the "%s" pack in the index.' % name_or_url)
215			return (pack['repo_url'], version)
216			else:
217			return (DownloadGitRepoAction._eval_repo_url(name_or_url), version)
218
219			@staticmethod
220			def _eval_repo_url(repo_url):
221			"""Allow passing short GitHub style URLs"""
222			if not repo_url:
223			raise Exception('No valid repo_url provided or could be inferred.')
224			has_git_extension = repo_url.endswith('.git')
225			if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
226			url = "https://github.com/{}".format(repo_url)
227			else:
228			url = repo_url
229			return url if has_git_extension else "{}.git".format(url)
230
231			@staticmethod
232			def _get_pack_name(pack_dir):
233			"""
234			Read pack name from the metadata file and sanitize it.
235			"""
236			with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file:
237			pack_meta = yaml.load(manifest_file)
238			return pack_meta['name'].replace(' ', '-').lower()
239
240			@staticmethod
241			def _get_gitref(repo, ref):
242			try:
243			return repo.commit(ref)
244			except (BadName, BadObject):
245			return False
246

Plexxi / st2

GitHub Access Token became invalid

Push — develop ( 6e30e7...c4b627 )

DownloadGitRepoAction B

Complexity

Size/Duplication

Importance

How to fix Complexity

Complex Class

Duplication Side-by-Side

Filter issues like