DownloadGitRepoAction._cleanup_repo() - Code Metrics - Inspection of "Pack management" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2842)

by Edward

created 2016-10-20 23:05 UTC

DownloadGitRepoAction._cleanup_repo() A

↳ Parent: DownloadGitRepoAction

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	2
c	1
b	0
f	0
dl	0
loc	5
rs	9.4285

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
import hashlib
import stat
import re

import six
import yaml
from git.repo import Repo
from gitdb.exc import BadName
from lockfile import LockFile

from st2common.runners.base_action import Action
from st2common.content import utils
from st2common.services.packs import get_pack_from_index
from st2common.util.green import shell

MANIFEST_FILE = 'pack.yaml'
CONFIG_FILE = 'config.yaml'
GITINFO_FILE = '.gitinfo'
PACK_RESERVE_CHARACTER = '.'
PACK_VERSION_SEPARATOR = '#'
SEMVER_REGEX = (r"^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)"
                r"(?:-[\da-z\-]+(?:\.[\da-z\-]+)*)?(?:\+[\da-z\-]+(?:\.[\da-z\-]+)*)?$")


class DownloadGitRepoAction(Action):
    def __init__(self, config=None, action_service=None):
        super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)

    def run(self, packs, abs_repo_base, verifyssl=True):
        result = {}

        for pack in packs:
            pack_url, pack_version = self._get_repo_url(pack)
            temp_dir = hashlib.md5(pack_url).hexdigest()

            with LockFile('/tmp/%s' % (temp_dir)):
                abs_local_path = self._clone_repo(temp_dir=temp_dir, repo_url=pack_url,
                                                  verifyssl=verifyssl, ref=pack_version)
                pack_name = self._get_pack_name(abs_local_path)
                try:
                    result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path)
                finally:
                    self._cleanup_repo(abs_local_path)

        return self._validate_result(result=result, repo_url=pack_url)

    @staticmethod
    def _clone_repo(temp_dir, repo_url, verifyssl=True, ref=None):
        user_home = os.path.expanduser('~')
        abs_local_path = os.path.join(user_home, temp_dir)

        # Switch to non-interactive mode
        os.environ['GIT_TERMINAL_PROMPT'] = '0'

        # Disable SSL cert checking if explictly asked
        if not verifyssl:
            os.environ['GIT_SSL_NO_VERIFY'] = 'true'

        if not ref:
            ref = 'master'

        # Clone the repo from git; we don't use shallow copying
        # because we want the user to work with the repo in the
        # future.
        repo = Repo.clone_from(repo_url, abs_local_path, branch='master')

        if not DownloadGitRepoAction._ref_exists(repo, ref):
            if re.match(SEMVER_REGEX, ref) and DownloadGitRepoAction._ref_exists(repo, "v%s" % ref):
                ref = "v%s" % ref
            else:
                raise ValueError("\"%s\" is not a valid ref in %s." % (ref, repo_url))

        repo.head.reference = repo.commit(ref)
        repo.head.reset(index=True, working_tree=True)

        return abs_local_path

    def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
        desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
        if desired:
            to = abs_repo_base
            dest_pack_path = os.path.join(abs_repo_base, pack_name)
            if os.path.exists(dest_pack_path):
                self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
                                  dest_pack_path)

                # Ensure to preserve any existing configuration
                old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
                new_config_file = os.path.join(abs_local_path, CONFIG_FILE)

                if os.path.isfile(old_config_file):
                    shutil.move(old_config_file, new_config_file)

                shutil.rmtree(dest_pack_path)

            self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
            shutil.move(abs_local_path, dest_pack_path)
            # post move fix all permissions.
            self._apply_pack_permissions(pack_path=dest_pack_path)
            message = 'Success.'
        elif message:
            message = 'Failure : %s' % message

        return (desired, message)

    def _apply_pack_permissions(self, pack_path):
        """
        Will recursively apply permission 770 to pack and its contents.
        """
        # 1. switch owner group to configured group
        pack_group = utils.get_pack_group()
        if pack_group:
            shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        # 2. Setup the right permissions and group ownership
        # These mask is same as mode = 775
        mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        os.chmod(pack_path, mode)

        # Yuck! Since os.chmod does not support chmod -R walk manually.
        for root, dirs, files in os.walk(pack_path):
            for d in dirs:
                os.chmod(os.path.join(root, d), mode)
            for f in files:
                os.chmod(os.path.join(root, f), mode)

    @staticmethod
    def _is_desired_pack(abs_pack_path, pack_name):
        # path has to exist.
        if not os.path.exists(abs_pack_path):
            return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
                    (pack_name))
        # should not include reserve characters
        if PACK_RESERVE_CHARACTER in pack_name:
            return (False, 'Pack name "%s" contains reserve character "%s"' %
                    (pack_name, PACK_RESERVE_CHARACTER))
        # must contain a manifest file. Empty file is ok for now.
        if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
            return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
        return (True, '')

    @staticmethod
    def _cleanup_repo(abs_local_path):
        # basic lock checking etc?
        if os.path.isdir(abs_local_path):
            shutil.rmtree(abs_local_path)

    @staticmethod
    def _validate_result(result, repo_url):
        atleast_one_success = False
        sanitized_result = {}
        for k, v in six.iteritems(result):
            atleast_one_success |= v[0]
            sanitized_result[k] = v[1]

        if not atleast_one_success:
            message_list = []
            message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
            message_list.append('Errors:')

            for pack, value in result.items():
                success, error = value
                message_list.append(error)

            message = '\n'.join(message_list)
            raise Exception(message)

        return sanitized_result

    @staticmethod
    def _get_repo_url(pack):
        pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
        name_or_url = pack_and_version[0]
        version = pack_and_version[1] if len(pack_and_version) > 1 else None

        if len(name_or_url.split('/')) == 1:
            pack = get_pack_from_index(name_or_url)
            if not pack:
                raise Exception('No record of the "%s" pack in the index.' % name_or_url)
            return (pack['repo_url'], version)
        else:
            return (DownloadGitRepoAction._eval_repo_url(name_or_url), version)

    @staticmethod
    def _eval_repo_url(repo_url):
        """Allow passing short GitHub style URLs"""
        if not repo_url:
            raise Exception('No valid repo_url provided or could be inferred.')
        has_git_extension = repo_url.endswith('.git')
        if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
            url = "https://github.com/{}".format(repo_url)
        else:
            url = repo_url
        return url if has_git_extension else "{}.git".format(url)

    @staticmethod
    def _get_pack_name(pack_dir):
        """
        Read pack name from the metadata file and sanitize it.
        """
        with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file:
            pack_meta = yaml.load(manifest_file)
        return pack_meta['name'].replace(' ', '-').lower()

    @staticmethod
    def _ref_exists(repo, ref):
        try:
            repo.commit(ref)
        except BadName:
            return False
        return True


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import os
17			import shutil
18			import hashlib
19			import stat
20			import re
21
22			import six
23			import yaml
24			from git.repo import Repo
25			from gitdb.exc import BadName
26			from lockfile import LockFile
27
28			from st2common.runners.base_action import Action
29			from st2common.content import utils
30			from st2common.services.packs import get_pack_from_index
31			from st2common.util.green import shell
32
33			MANIFEST_FILE = 'pack.yaml'
34			CONFIG_FILE = 'config.yaml'
35			GITINFO_FILE = '.gitinfo'
36			PACK_RESERVE_CHARACTER = '.'
37			PACK_VERSION_SEPARATOR = '#'
38			SEMVER_REGEX = (r"^(?:0\|[1-9]\d)\.(?:0\|[1-9]\d)\.(?:0\|[1-9]\d*)"
39			r"(?:-[\da-z\-]+(?:\.[\da-z\-]+))?(?:\+[\da-z\-]+(?:\.[\da-z\-]+))?$")
40
41
42			class DownloadGitRepoAction(Action):
43			def __init__(self, config=None, action_service=None):
44			super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service)
45
46			def run(self, packs, abs_repo_base, verifyssl=True):
47			result = {}
48
49			for pack in packs:
50			pack_url, pack_version = self._get_repo_url(pack)
51			temp_dir = hashlib.md5(pack_url).hexdigest()
52
53			with LockFile('/tmp/%s' % (temp_dir)):
54			abs_local_path = self._clone_repo(temp_dir=temp_dir, repo_url=pack_url,
55			verifyssl=verifyssl, ref=pack_version)
56			pack_name = self._get_pack_name(abs_local_path)
57			try:
58			result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path)
59			finally:
60			self._cleanup_repo(abs_local_path)
61
62			return self._validate_result(result=result, repo_url=pack_url)
63
64			@staticmethod
65			def _clone_repo(temp_dir, repo_url, verifyssl=True, ref=None):
66			user_home = os.path.expanduser('~')
67			abs_local_path = os.path.join(user_home, temp_dir)
68
69			# Switch to non-interactive mode
70			os.environ['GIT_TERMINAL_PROMPT'] = '0'
71
72			# Disable SSL cert checking if explictly asked
73			if not verifyssl:
74			os.environ['GIT_SSL_NO_VERIFY'] = 'true'
75
76			if not ref:
77			ref = 'master'
78
79			# Clone the repo from git; we don't use shallow copying
80			# because we want the user to work with the repo in the
81			# future.
82			repo = Repo.clone_from(repo_url, abs_local_path, branch='master')
83
84			if not DownloadGitRepoAction._ref_exists(repo, ref):
85			if re.match(SEMVER_REGEX, ref) and DownloadGitRepoAction._ref_exists(repo, "v%s" % ref):
86			ref = "v%s" % ref
87			else:
88			raise ValueError("\"%s\" is not a valid ref in %s." % (ref, repo_url))
89
90			repo.head.reference = repo.commit(ref)
91			repo.head.reset(index=True, working_tree=True)
92
93			return abs_local_path
94
95			def _move_pack(self, abs_repo_base, pack_name, abs_local_path):
96			desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name)
97			if desired:
98			to = abs_repo_base
99			dest_pack_path = os.path.join(abs_repo_base, pack_name)
100			if os.path.exists(dest_pack_path):
101			self.logger.debug('Removing existing pack %s in %s to replace.', pack_name,
102			dest_pack_path)
103
104			# Ensure to preserve any existing configuration
105			old_config_file = os.path.join(dest_pack_path, CONFIG_FILE)
106			new_config_file = os.path.join(abs_local_path, CONFIG_FILE)
107
108			if os.path.isfile(old_config_file):
109			shutil.move(old_config_file, new_config_file)
110
111			shutil.rmtree(dest_pack_path)
112
113			self.logger.debug('Moving pack from %s to %s.', abs_local_path, to)
114			shutil.move(abs_local_path, dest_pack_path)
115			# post move fix all permissions.
116			self._apply_pack_permissions(pack_path=dest_pack_path)
117			message = 'Success.'
118			elif message:
119			message = 'Failure : %s' % message
120
121			return (desired, message)
122
123			def _apply_pack_permissions(self, pack_path):
124			"""
125			Will recursively apply permission 770 to pack and its contents.
126			"""
127			# 1. switch owner group to configured group
128			pack_group = utils.get_pack_group()
129			if pack_group:
130			shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
131
132			# 2. Setup the right permissions and group ownership
133			# These mask is same as mode = 775
134			mode = stat.S_IRWXU \| stat.S_IRWXG \| stat.S_IROTH \| stat.S_IXOTH
135			os.chmod(pack_path, mode)
136
137			# Yuck! Since os.chmod does not support chmod -R walk manually.
138			for root, dirs, files in os.walk(pack_path):
139			for d in dirs:
140			os.chmod(os.path.join(root, d), mode)
141			for f in files:
142			os.chmod(os.path.join(root, f), mode)
143
144			@staticmethod
145			def _is_desired_pack(abs_pack_path, pack_name):
146			# path has to exist.
147			if not os.path.exists(abs_pack_path):
148			return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' %
149			(pack_name))
150			# should not include reserve characters
151			if PACK_RESERVE_CHARACTER in pack_name:
152			return (False, 'Pack name "%s" contains reserve character "%s"' %
153			(pack_name, PACK_RESERVE_CHARACTER))
154			# must contain a manifest file. Empty file is ok for now.
155			if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)):
156			return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE))
157			return (True, '')
158
159			@staticmethod
160			def _cleanup_repo(abs_local_path):
161			# basic lock checking etc?
162			if os.path.isdir(abs_local_path):
163			shutil.rmtree(abs_local_path)
164
165			@staticmethod
166			def _validate_result(result, repo_url):
167			atleast_one_success = False
168			sanitized_result = {}
169			for k, v in six.iteritems(result):
170			atleast_one_success \|= v[0]
171			sanitized_result[k] = v[1]
172
173			if not atleast_one_success:
174			message_list = []
175			message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url))
176			message_list.append('Errors:')
177
178			for pack, value in result.items():
179			success, error = value
180			message_list.append(error)
181
182			message = '\n'.join(message_list)
183			raise Exception(message)
184
185			return sanitized_result
186
187			@staticmethod
188			def _get_repo_url(pack):
189			pack_and_version = pack.split(PACK_VERSION_SEPARATOR)
190			name_or_url = pack_and_version[0]
191			version = pack_and_version[1] if len(pack_and_version) > 1 else None
192
193			if len(name_or_url.split('/')) == 1:
194			pack = get_pack_from_index(name_or_url)
195			if not pack:
196			raise Exception('No record of the "%s" pack in the index.' % name_or_url)
197			return (pack['repo_url'], version)
198			else:
199			return (DownloadGitRepoAction._eval_repo_url(name_or_url), version)
200
201			@staticmethod
202			def _eval_repo_url(repo_url):
203			"""Allow passing short GitHub style URLs"""
204			if not repo_url:
205			raise Exception('No valid repo_url provided or could be inferred.')
206			has_git_extension = repo_url.endswith('.git')
207			if len(repo_url.split('/')) == 2 and "git@" not in repo_url:
208			url = "https://github.com/{}".format(repo_url)
209			else:
210			url = repo_url
211			return url if has_git_extension else "{}.git".format(url)
212
213			@staticmethod
214			def _get_pack_name(pack_dir):
215			"""
216			Read pack name from the metadata file and sanitize it.
217			"""
218			with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file:
219			pack_meta = yaml.load(manifest_file)
220			return pack_meta['name'].replace(' ', '-').lower()
221
222			@staticmethod
223			def _ref_exists(repo, ref):
224			try:
225			repo.commit(ref)
226			except BadName:
227			return False
228			return True
229

StackStorm / st2

Pull Request — master (#2842)

DownloadGitRepoAction._cleanup_repo() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like