DownloadGitRepoAction._get_pack_name_and_url() - Code Metrics - Inspection of "[WIP] [NSFW] Make packs.install assume StackStorm..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2836)

by Edward

created 2016-07-27 11:16 UTC

DownloadGitRepoAction._get_pack_name_and_url() A

↳ Parent: DownloadGitRepoAction

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	2
c	2
b	0
f	0
dl	0
loc	7
rs	9.4285

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
import hashlib
import stat

from git.repo import Repo
from lockfile import LockFile

from st2actions.runners.pythonrunner import Action
from st2common.util.green import shell

MANIFEST_FILE = 'pack.yaml'
CONFIG_FILE = 'config.yaml'
GITINFO_FILE = '.gitinfo'
PACK_RESERVE_CHARACTER = '.'

PACK_GROUP_CFG_KEY = 'pack_group'
EXCHANGE_URL_KEY = 'exchange_url'


class DownloadGitRepoAction(Action):
    def __init__(self, config=None, action_service=None):
        super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)
        self._subtree = None
        self._repo_url = None

    def run(self, name_or_url, ref, abs_repo_base, verifyssl=True, branch='master'):

        self._pack_name, self._pack_url = self._get_pack_name_and_url(
            name_or_url,
            self.config.get(EXCHANGE_URL_KEY, None)
        )

        lock_name = hashlib.md5(self._pack_name).hexdigest() + '.lock'

        with LockFile('/tmp/%s' % (lock_name)):
            abs_local_path = self._clone_repo(repo_name=self._pack_name, repo_url=self._pack_url,
                                              verifyssl=verifyssl, branch=ref)
            try:
                result = self._move_pack(abs_repo_base, self._pack_name, abs_local_path)
            finally:
                self._cleanup_repo(abs_local_path)

        return self._validate_result(result=result, repo_url=self._pack_url)

    @staticmethod
    def _clone_repo(repo_name, repo_url, verifyssl=True, branch='master'):
        user_home = os.path.expanduser('~')
        abs_local_path = os.path.join(user_home, repo_name)

        # Disable SSL cert checking if explictly asked
        if not verifyssl:
            os.environ['GIT_SSL_NO_VERIFY'] = 'true'
        # Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
        # specific branch so save some download time.
        Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
        return abs_local_path

    def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
        result = {}

        desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
        if desired:
            to = abs_repo_base
            dest_pack_path = os.path.join(abs_repo_base, pack_name)
            if os.path.exists(dest_pack_path):
                self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
                                  dest_pack_path)

                # Ensure to preserve any existing configuration
                old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
                new_config_file = os.path.join(abs_local_path, CONFIG_FILE)

                if os.path.isfile(old_config_file):
                    shutil.move(old_config_file, new_config_file)

                shutil.rmtree(dest_pack_path)

            self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
            shutil.move(abs_local_path, to)
            # post move fix all permissions.
            self._apply_pack_permissions(pack_path=dest_pack_path)
            message = 'Success.'
        elif message:
            message = 'Failure : %s' % message

        result[pack_name] = (desired, message)

        return result

    def _apply_pack_permissions(self, pack_path):
        """
        Will recursively apply permission 770 to pack and its contents.
        """
        # 1. switch owner group to configured group
        pack_group = self.config.get(PACK_GROUP_CFG_KEY, None)
        if pack_group:
            shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        # 2. Setup the right permissions and group ownership
        # These mask is same as mode = 775
        mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        os.chmod(pack_path, mode)

        # Yuck! Since os.chmod does not support chmod -R walk manually.
        for root, dirs, files in os.walk(pack_path):
            for d in dirs:
                os.chmod(os.path.join(root, d), mode)
            for f in files:
                os.chmod(os.path.join(root, f), mode)

    @staticmethod
    def _is_desired_pack(abs_pack_path, pack_name):
        # path has to exist.      

        if not os.path.exists(abs_pack_path):     

            return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %       

                    (pack_name))      

        # should not include reserve characters
        if PACK_RESERVE_CHARACTER in pack_name:
            return (False, 'Pack name "%s" contains reserve character "%s"' %
                    (pack_name, PACK_RESERVE_CHARACTER))
        # must contain a manifest file. Empty file is ok for now.
        if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
            return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
        return (True, '')

    @staticmethod
    def _cleanup_repo(abs_local_path):
        # basic lock checking etc?
        if os.path.isdir(abs_local_path):
            shutil.rmtree(abs_local_path)

    @staticmethod
    def _validate_result(result, repo_url):
        atleast_one_success = False
        sanitized_result = {}
        for k, v in six.iteritems(result):

            atleast_one_success |= v[0]
            sanitized_result[k] = v[1]

        if not atleast_one_success:
            message_list = []
            message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
            message_list.append('Errors:')

            for pack, value in result.items():
                success, error = value
                message_list.append(error)

            message = '\n'.join(message_list)
            raise Exception(message)

        return sanitized_result

    @staticmethod
    def _get_pack_name_and_url(name_or_url, exchange_url):
        if len(name_or_url.split('/')) == 1:
            return (name_or_url, "{}/{}.git".format(exchange_url, name_or_url))
        else:
            return (DownloadGitRepoAction._eval_repo_name(name_or_url),
                    DownloadGitRepoAction._eval_repo_url(name_or_url))

    @staticmethod
    def _eval_repo_url(repo_url):
        """Allow passing short GitHub style URLs"""
        if not repo_url:
            raise Exception('No valid repo_url provided or could be inferred.')
        has_git_extension = repo_url.endswith('.git')
        if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
            url = "https://github.com/{}".format(repo_url)
        else:
            url = repo_url
        return url if has_git_extension else "{}.git".format(url)

    @staticmethod
    def _eval_repo_name(repo_url):
        """
        Evaluate the name of the repo.
        https://github.com/StackStorm/st2contrib.git -> st2contrib
        https://github.com/StackStorm/st2contrib -> st2contrib
        [email protected]:StackStorm/st2contrib.git -> st2contrib
        [email protected]:StackStorm/st2contrib -> st2contrib
        """
        last_forward_slash = repo_url.rfind('/')
        next_dot = repo_url.find('.', last_forward_slash)
        # If dot does not follow last_forward_slash return till the end
        if next_dot < last_forward_slash:
            return repo_url[last_forward_slash + 1:]
        return repo_url[last_forward_slash + 1:next_dot]


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import os
17			import shutil
18			import hashlib
19			import stat
20
21			from git.repo import Repo
22			from lockfile import LockFile
23
24			from st2actions.runners.pythonrunner import Action
25			from st2common.util.green import shell
26
27			MANIFEST_FILE = 'pack.yaml'
28			CONFIG_FILE = 'config.yaml'
29			GITINFO_FILE = '.gitinfo'
30			PACK_RESERVE_CHARACTER = '.'
31
32			PACK_GROUP_CFG_KEY = 'pack_group'
33			EXCHANGE_URL_KEY = 'exchange_url'
34
35
36			class DownloadGitRepoAction(Action):
37			def __init__(self, config=None, action_service=None):
38			super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)
39			self._subtree = None
40			self._repo_url = None
41
42			def run(self, name_or_url, ref, abs_repo_base, verifyssl=True, branch='master'):
43
44			self._pack_name, self._pack_url = self._get_pack_name_and_url(
45			name_or_url,
46			self.config.get(EXCHANGE_URL_KEY, None)
47			)
48
49			lock_name = hashlib.md5(self._pack_name).hexdigest() + '.lock'
50
51			with LockFile('/tmp/%s' % (lock_name)):
52			abs_local_path = self._clone_repo(repo_name=self._pack_name, repo_url=self._pack_url,
53			verifyssl=verifyssl, branch=ref)
54			try:
55			result = self._move_pack(abs_repo_base, self._pack_name, abs_local_path)
56			finally:
57			self._cleanup_repo(abs_local_path)
58
59			return self._validate_result(result=result, repo_url=self._pack_url)
60
61			@staticmethod
62			def _clone_repo(repo_name, repo_url, verifyssl=True, branch='master'):
63			user_home = os.path.expanduser('~')
64			abs_local_path = os.path.join(user_home, repo_name)
65
66			# Disable SSL cert checking if explictly asked
67			if not verifyssl:
68			os.environ['GIT_SSL_NO_VERIFY'] = 'true'
69			# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
70			# specific branch so save some download time.
71			Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
72			return abs_local_path
73
74			def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
75			result = {}
76
77			desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
78			if desired:
79			to = abs_repo_base
80			dest_pack_path = os.path.join(abs_repo_base, pack_name)
81			if os.path.exists(dest_pack_path):
82			self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
83			dest_pack_path)
84
85			# Ensure to preserve any existing configuration
86			old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
87			new_config_file = os.path.join(abs_local_path, CONFIG_FILE)
88
89			if os.path.isfile(old_config_file):
90			shutil.move(old_config_file, new_config_file)
91
92			shutil.rmtree(dest_pack_path)
93
94			self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
95			shutil.move(abs_local_path, to)
96			# post move fix all permissions.
97			self._apply_pack_permissions(pack_path=dest_pack_path)
98			message = 'Success.'
99			elif message:
100			message = 'Failure : %s' % message
101
102			result[pack_name] = (desired, message)
103
104			return result
105
106			def _apply_pack_permissions(self, pack_path):
107			"""
108			Will recursively apply permission 770 to pack and its contents.
109			"""
110			# 1. switch owner group to configured group
111			pack_group = self.config.get(PACK_GROUP_CFG_KEY, None)
112			if pack_group:
113			shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
114
115			# 2. Setup the right permissions and group ownership
116			# These mask is same as mode = 775
117			mode = stat.S_IRWXU \| stat.S_IRWXG \| stat.S_IROTH \| stat.S_IXOTH
118			os.chmod(pack_path, mode)
119
120			# Yuck! Since os.chmod does not support chmod -R walk manually.
121			for root, dirs, files in os.walk(pack_path):
122			for d in dirs:
123			os.chmod(os.path.join(root, d), mode)
124			for f in files:
125			os.chmod(os.path.join(root, f), mode)
126
127			@staticmethod
128			def _is_desired_pack(abs_pack_path, pack_name):
129			# path has to exist.
			0 ignored issues – show Coding Style introduced 2016-07-27 11:26 UTC by Report Bug Copy Issue Report Trailing whitespace Loading history...
130			if not os.path.exists(abs_pack_path):
			0 ignored issues – show Coding Style introduced 2016-07-27 11:26 UTC by Report Bug Copy Issue Report Trailing whitespace Loading history...
131			return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
			0 ignored issues – show Coding Style introduced 2016-07-27 11:26 UTC by Report Bug Copy Issue Report Trailing whitespace Loading history...
132			(pack_name))
			0 ignored issues – show Coding Style introduced 2016-07-27 11:26 UTC by Report Bug Copy Issue Report Trailing whitespace Loading history...
133			# should not include reserve characters
134			if PACK_RESERVE_CHARACTER in pack_name:
135			return (False, 'Pack name "%s" contains reserve character "%s"' %
136			(pack_name, PACK_RESERVE_CHARACTER))
137			# must contain a manifest file. Empty file is ok for now.
138			if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
139			return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
140			return (True, '')
141
142			@staticmethod
143			def _cleanup_repo(abs_local_path):
144			# basic lock checking etc?
145			if os.path.isdir(abs_local_path):
146			shutil.rmtree(abs_local_path)
147
148			@staticmethod
149			def _validate_result(result, repo_url):
150			atleast_one_success = False
151			sanitized_result = {}
152			for k, v in six.iteritems(result):
			0 ignored issues – show Comprehensibility Best Practice introduced 2016-07-27 11:19 UTC by Report Bug Copy Issue Report Undefined variable 'six' Loading history...
153			atleast_one_success \|= v[0]
154			sanitized_result[k] = v[1]
155
156			if not atleast_one_success:
157			message_list = []
158			message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
159			message_list.append('Errors:')
160
161			for pack, value in result.items():
162			success, error = value
163			message_list.append(error)
164
165			message = '\n'.join(message_list)
166			raise Exception(message)
167
168			return sanitized_result
169
170			@staticmethod
171			def _get_pack_name_and_url(name_or_url, exchange_url):
172			if len(name_or_url.split('/')) == 1:
173			return (name_or_url, "{}/{}.git".format(exchange_url, name_or_url))
174			else:
175			return (DownloadGitRepoAction._eval_repo_name(name_or_url),
176			DownloadGitRepoAction._eval_repo_url(name_or_url))
177
178			@staticmethod
179			def _eval_repo_url(repo_url):
180			"""Allow passing short GitHub style URLs"""
181			if not repo_url:
182			raise Exception('No valid repo_url provided or could be inferred.')
183			has_git_extension = repo_url.endswith('.git')
184			if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
185			url = "https://github.com/{}".format(repo_url)
186			else:
187			url = repo_url
188			return url if has_git_extension else "{}.git".format(url)
189
190			@staticmethod
191			def _eval_repo_name(repo_url):
192			"""
193			Evaluate the name of the repo.
194			https://github.com/StackStorm/st2contrib.git -> st2contrib
195			https://github.com/StackStorm/st2contrib -> st2contrib
196			[email protected]:StackStorm/st2contrib.git -> st2contrib
197			[email protected]:StackStorm/st2contrib -> st2contrib
198			"""
199			last_forward_slash = repo_url.rfind('/')
200			next_dot = repo_url.find('.', last_forward_slash)
201			# If dot does not follow last_forward_slash return till the end
202			if next_dot < last_forward_slash:
203			return repo_url[last_forward_slash + 1:]
204			return repo_url[last_forward_slash + 1:next_dot]
205

StackStorm / st2

Pull Request — master (#2836)

DownloadGitRepoAction._get_pack_name_and_url() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like