DownloadGitRepoAction._get_repo_url() - Code Metrics - Inspection of "Pack management" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2842)

by Edward

created 2016-10-18 10:20 UTC

DownloadGitRepoAction._get_repo_url() A

↳ Parent: DownloadGitRepoAction

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	4
c	1
b	0
f	0
dl	0
loc	13
rs	9.2

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
import hashlib
import stat

import six
import yaml
from git.repo import Repo
from lockfile import LockFile

from st2common.runners.base_action import Action
from st2common.content import utils
from st2common.services.packs import get_pack_from_index
from st2common.util.green import shell

MANIFEST_FILE = 'pack.yaml'
CONFIG_FILE = 'config.yaml'
GITINFO_FILE = '.gitinfo'
PACK_RESERVE_CHARACTER = '.'
PACK_VERSION_SEPARATOR = '#'


class DownloadGitRepoAction(Action):
    def __init__(self, config=None, action_service=None):
        super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)

    def run(self, packs, abs_repo_base, verifyssl=True):
        result = {}

        for pack in packs:
            pack_url, pack_version = self._get_repo_url(pack)
            temp_dir = hashlib.md5(pack_url).hexdigest()

            with LockFile('/tmp/%s' % (temp_dir)):
                abs_local_path = self._clone_repo(temp_dir=temp_dir, repo_url=pack_url,
                                                  verifyssl=verifyssl, branch=pack_version)
                pack_name = self._read_pack_name(abs_local_path)
                try:
                    result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path)
                finally:
                    self._cleanup_repo(abs_local_path)

        return self._validate_result(result=result, repo_url=pack_url)

    @staticmethod
    def _clone_repo(temp_dir, repo_url, verifyssl=True, branch=None):
        user_home = os.path.expanduser('~')
        abs_local_path = os.path.join(user_home, temp_dir)

        # Switch to non-interactive mode
        os.environ['GIT_TERMINAL_PROMPT'] = '0'

        # Disable SSL cert checking if explictly asked
        if not verifyssl:
            os.environ['GIT_SSL_NO_VERIFY'] = 'true'

        if not branch:
            branch = 'master'

        # Clone the repo from git; we don't use shallow copying
        # because we want the user to work with the repo in the
        # future.
        Repo.clone_from(repo_url, abs_local_path, branch=branch)
        return abs_local_path

    def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
        desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
        if desired:
            to = abs_repo_base
            dest_pack_path = os.path.join(abs_repo_base, pack_name)
            if os.path.exists(dest_pack_path):
                self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
                                  dest_pack_path)

                # Ensure to preserve any existing configuration
                old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
                new_config_file = os.path.join(abs_local_path, CONFIG_FILE)

                if os.path.isfile(old_config_file):
                    shutil.move(old_config_file, new_config_file)

                shutil.rmtree(dest_pack_path)

            self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
            shutil.move(abs_local_path, dest_pack_path)
            # post move fix all permissions.
            self._apply_pack_permissions(pack_path=dest_pack_path)
            message = 'Success.'
        elif message:
            message = 'Failure : %s' % message

        return (desired, message)

    def _apply_pack_permissions(self, pack_path):
        """
        Will recursively apply permission 770 to pack and its contents.
        """
        # 1. switch owner group to configured group
        pack_group = utils.get_pack_group()
        if pack_group:
            shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        # 2. Setup the right permissions and group ownership
        # These mask is same as mode = 775
        mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        os.chmod(pack_path, mode)

        # Yuck! Since os.chmod does not support chmod -R walk manually.
        for root, dirs, files in os.walk(pack_path):
            for d in dirs:
                os.chmod(os.path.join(root, d), mode)
            for f in files:
                os.chmod(os.path.join(root, f), mode)

    @staticmethod
    def _is_desired_pack(abs_pack_path, pack_name):
        # path has to exist.
        if not os.path.exists(abs_pack_path):
            return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
                    (pack_name))
        # should not include reserve characters
        if PACK_RESERVE_CHARACTER in pack_name:
            return (False, 'Pack name "%s" contains reserve character "%s"' %
                    (pack_name, PACK_RESERVE_CHARACTER))
        # must contain a manifest file. Empty file is ok for now.
        if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
            return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
        return (True, '')

    @staticmethod
    def _cleanup_repo(abs_local_path):
        # basic lock checking etc?
        if os.path.isdir(abs_local_path):
            shutil.rmtree(abs_local_path)

    @staticmethod
    def _validate_result(result, repo_url):
        atleast_one_success = False
        sanitized_result = {}
        for k, v in six.iteritems(result):
            atleast_one_success |= v[0]
            sanitized_result[k] = v[1]

        if not atleast_one_success:
            message_list = []
            message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
            message_list.append('Errors:')

            for pack, value in result.items():
                success, error = value
                message_list.append(error)

            message = '\n'.join(message_list)
            raise Exception(message)

        return sanitized_result

    @staticmethod
    def _get_repo_url(pack):
        pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
        name_or_url = pack_and_version[0]
        version = pack_and_version[1] if len(pack_and_version) > 1 else None

        if len(name_or_url.split('/')) == 1:
            pack = get_pack_from_index(name_or_url)
            if not pack:
                raise Exception('No record of the "%s" pack in the index.' % name_or_url)
            return (pack.repo_url, version)
        else:
            return (DownloadGitRepoAction._eval_repo_url(name_or_url), version)

    @staticmethod
    def _eval_repo_url(repo_url):
        """Allow passing short GitHub style URLs"""
        if not repo_url:
            raise Exception('No valid repo_url provided or could be inferred.')
        has_git_extension = repo_url.endswith('.git')
        if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
            url = "https://github.com/{}".format(repo_url)
        else:
            url = repo_url
        return url if has_git_extension else "{}.git".format(url)

    @staticmethod
    def _read_pack_name(pack_dir):
        """
        Read pack name from the metadata file.
        """
        with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file:
            pack_meta = yaml.load(manifest_file)
        return pack_meta['name']


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import os
17			import shutil
18			import hashlib
19			import stat
20
21			import six
22			import yaml
23			from git.repo import Repo
24			from lockfile import LockFile
25
26			from st2common.runners.base_action import Action
27			from st2common.content import utils
28			from st2common.services.packs import get_pack_from_index
29			from st2common.util.green import shell
30
31			MANIFEST_FILE = 'pack.yaml'
32			CONFIG_FILE = 'config.yaml'
33			GITINFO_FILE = '.gitinfo'
34			PACK_RESERVE_CHARACTER = '.'
35			PACK_VERSION_SEPARATOR = '#'
36
37
38			class DownloadGitRepoAction(Action):
39			def __init__(self, config=None, action_service=None):
40			super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)
41
42			def run(self, packs, abs_repo_base, verifyssl=True):
43			result = {}
44
45			for pack in packs:
46			pack_url, pack_version = self._get_repo_url(pack)
47			temp_dir = hashlib.md5(pack_url).hexdigest()
48
49			with LockFile('/tmp/%s' % (temp_dir)):
50			abs_local_path = self._clone_repo(temp_dir=temp_dir, repo_url=pack_url,
51			verifyssl=verifyssl, branch=pack_version)
52			pack_name = self._read_pack_name(abs_local_path)
53			try:
54			result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path)
55			finally:
56			self._cleanup_repo(abs_local_path)
57
58			return self._validate_result(result=result, repo_url=pack_url)
59
60			@staticmethod
61			def _clone_repo(temp_dir, repo_url, verifyssl=True, branch=None):
62			user_home = os.path.expanduser('~')
63			abs_local_path = os.path.join(user_home, temp_dir)
64
65			# Switch to non-interactive mode
66			os.environ['GIT_TERMINAL_PROMPT'] = '0'
67
68			# Disable SSL cert checking if explictly asked
69			if not verifyssl:
70			os.environ['GIT_SSL_NO_VERIFY'] = 'true'
71
72			if not branch:
73			branch = 'master'
74
75			# Clone the repo from git; we don't use shallow copying
76			# because we want the user to work with the repo in the
77			# future.
78			Repo.clone_from(repo_url, abs_local_path, branch=branch)
79			return abs_local_path
80
81			def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
82			desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
83			if desired:
84			to = abs_repo_base
85			dest_pack_path = os.path.join(abs_repo_base, pack_name)
86			if os.path.exists(dest_pack_path):
87			self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
88			dest_pack_path)
89
90			# Ensure to preserve any existing configuration
91			old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
92			new_config_file = os.path.join(abs_local_path, CONFIG_FILE)
93
94			if os.path.isfile(old_config_file):
95			shutil.move(old_config_file, new_config_file)
96
97			shutil.rmtree(dest_pack_path)
98
99			self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
100			shutil.move(abs_local_path, dest_pack_path)
101			# post move fix all permissions.
102			self._apply_pack_permissions(pack_path=dest_pack_path)
103			message = 'Success.'
104			elif message:
105			message = 'Failure : %s' % message
106
107			return (desired, message)
108
109			def _apply_pack_permissions(self, pack_path):
110			"""
111			Will recursively apply permission 770 to pack and its contents.
112			"""
113			# 1. switch owner group to configured group
114			pack_group = utils.get_pack_group()
115			if pack_group:
116			shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
117
118			# 2. Setup the right permissions and group ownership
119			# These mask is same as mode = 775
120			mode = stat.S_IRWXU \| stat.S_IRWXG \| stat.S_IROTH \| stat.S_IXOTH
121			os.chmod(pack_path, mode)
122
123			# Yuck! Since os.chmod does not support chmod -R walk manually.
124			for root, dirs, files in os.walk(pack_path):
125			for d in dirs:
126			os.chmod(os.path.join(root, d), mode)
127			for f in files:
128			os.chmod(os.path.join(root, f), mode)
129
130			@staticmethod
131			def _is_desired_pack(abs_pack_path, pack_name):
132			# path has to exist.
133			if not os.path.exists(abs_pack_path):
134			return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
135			(pack_name))
136			# should not include reserve characters
137			if PACK_RESERVE_CHARACTER in pack_name:
138			return (False, 'Pack name "%s" contains reserve character "%s"' %
139			(pack_name, PACK_RESERVE_CHARACTER))
140			# must contain a manifest file. Empty file is ok for now.
141			if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
142			return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
143			return (True, '')
144
145			@staticmethod
146			def _cleanup_repo(abs_local_path):
147			# basic lock checking etc?
148			if os.path.isdir(abs_local_path):
149			shutil.rmtree(abs_local_path)
150
151			@staticmethod
152			def _validate_result(result, repo_url):
153			atleast_one_success = False
154			sanitized_result = {}
155			for k, v in six.iteritems(result):
156			atleast_one_success \|= v[0]
157			sanitized_result[k] = v[1]
158
159			if not atleast_one_success:
160			message_list = []
161			message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
162			message_list.append('Errors:')
163
164			for pack, value in result.items():
165			success, error = value
166			message_list.append(error)
167
168			message = '\n'.join(message_list)
169			raise Exception(message)
170
171			return sanitized_result
172
173			@staticmethod
174			def _get_repo_url(pack):
175			pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
176			name_or_url = pack_and_version[0]
177			version = pack_and_version[1] if len(pack_and_version) > 1 else None
178
179			if len(name_or_url.split('/')) == 1:
180			pack = get_pack_from_index(name_or_url)
181			if not pack:
182			raise Exception('No record of the "%s" pack in the index.' % name_or_url)
183			return (pack.repo_url, version)
184			else:
185			return (DownloadGitRepoAction._eval_repo_url(name_or_url), version)
186
187			@staticmethod
188			def _eval_repo_url(repo_url):
189			"""Allow passing short GitHub style URLs"""
190			if not repo_url:
191			raise Exception('No valid repo_url provided or could be inferred.')
192			has_git_extension = repo_url.endswith('.git')
193			if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
194			url = "https://github.com/{}".format(repo_url)
195			else:
196			url = repo_url
197			return url if has_git_extension else "{}.git".format(url)
198
199			@staticmethod
200			def _read_pack_name(pack_dir):
201			"""
202			Read pack name from the metadata file.
203			"""
204			with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file:
205			pack_meta = yaml.load(manifest_file)
206			return pack_meta['name']
207

StackStorm / st2

Pull Request — master (#2842)

DownloadGitRepoAction._get_repo_url() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like