|
1
|
|
|
# Licensed to the StackStorm, Inc ('StackStorm') under one or more |
|
2
|
|
|
# contributor license agreements. See the NOTICE file distributed with |
|
3
|
|
|
# this work for additional information regarding copyright ownership. |
|
4
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
5
|
|
|
# (the "License"); you may not use this file except in compliance with |
|
6
|
|
|
# the License. You may obtain a copy of the License at |
|
7
|
|
|
# |
|
8
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0 |
|
9
|
|
|
# |
|
10
|
|
|
# Unless required by applicable law or agreed to in writing, software |
|
11
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS, |
|
12
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
13
|
|
|
# See the License for the specific language governing permissions and |
|
14
|
|
|
# limitations under the License. |
|
15
|
|
|
|
|
16
|
|
|
import os |
|
17
|
|
|
import shutil |
|
18
|
|
|
import hashlib |
|
19
|
|
|
import stat |
|
20
|
|
|
|
|
21
|
|
|
import six |
|
22
|
|
|
from git.repo import Repo |
|
23
|
|
|
from lockfile import LockFile |
|
24
|
|
|
|
|
25
|
|
|
from st2actions.runners.pythonrunner import Action |
|
26
|
|
|
from st2common.util.green import shell |
|
27
|
|
|
|
|
28
|
|
|
|
|
29
|
|
|
MANIFEST_FILE = 'pack.yaml' |
|
30
|
|
|
CONFIG_FILE = 'config.yaml' |
|
31
|
|
|
GITINFO_FILE = '.gitinfo' |
|
32
|
|
|
PACK_RESERVE_CHARACTER = '.' |
|
33
|
|
|
|
|
34
|
|
|
|
|
35
|
|
|
PACK_GROUP_CFG_KEY = 'pack_group' |
|
36
|
|
|
EXCHANGE_URL_KEY = 'exchange_url' |
|
37
|
|
|
|
|
38
|
|
|
|
|
39
|
|
|
class DownloadGitRepoAction(Action): |
|
40
|
|
|
def __init__(self, config=None, action_service=None): |
|
41
|
|
|
super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service) |
|
42
|
|
|
self._subtree = None |
|
43
|
|
|
self._repo_url = None |
|
44
|
|
|
|
|
45
|
|
|
def run(self, name_or_url, ref, abs_repo_base, verifyssl=True, branch='master'): |
|
46
|
|
|
|
|
47
|
|
|
self._pack_name, self._pack_url = self._get_pack_name_and_url( |
|
48
|
|
|
name_or_url, |
|
49
|
|
|
self.config.get(EXCHANGE_URL_KEY, None) |
|
50
|
|
|
) |
|
51
|
|
|
|
|
52
|
|
|
lock_name = hashlib.md5(self._pack_name).hexdigest() + '.lock' |
|
53
|
|
|
|
|
54
|
|
|
with LockFile('/tmp/%s' % (lock_name)): |
|
55
|
|
|
abs_local_path = self._clone_repo(repo_name=self._pack_name, repo_url=self._pack_url, |
|
56
|
|
|
verifyssl=verifyssl, branch=ref) |
|
57
|
|
|
try: |
|
58
|
|
|
result = self._move_pack(abs_repo_base, self._pack_name, abs_local_path) |
|
59
|
|
|
finally: |
|
60
|
|
|
self._cleanup_repo(abs_local_path) |
|
61
|
|
|
|
|
62
|
|
|
return self._validate_result(result=result, repo_url=self._pack_url) |
|
63
|
|
|
|
|
64
|
|
|
@staticmethod |
|
65
|
|
|
def _clone_repo(repo_name, repo_url, verifyssl=True, branch='master'): |
|
66
|
|
|
user_home = os.path.expanduser('~') |
|
67
|
|
|
abs_local_path = os.path.join(user_home, repo_name) |
|
68
|
|
|
|
|
69
|
|
|
# Disable SSL cert checking if explictly asked |
|
70
|
|
|
if not verifyssl: |
|
71
|
|
|
os.environ['GIT_SSL_NO_VERIFY'] = 'true' |
|
72
|
|
|
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a |
|
73
|
|
|
# specific branch so save some download time. |
|
74
|
|
|
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1) |
|
75
|
|
|
return abs_local_path |
|
76
|
|
|
|
|
77
|
|
|
def _apply_pack_permissions(self, pack_path): |
|
78
|
|
|
""" |
|
79
|
|
|
Will recursively apply permission 770 to pack and its contents. |
|
80
|
|
|
""" |
|
81
|
|
|
# 1. switch owner group to configured group |
|
82
|
|
|
pack_group = self.config.get(PACK_GROUP_CFG_KEY, None) |
|
83
|
|
|
if pack_group: |
|
84
|
|
|
shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) |
|
85
|
|
|
|
|
86
|
|
|
# 2. Setup the right permissions and group ownership |
|
87
|
|
|
# These mask is same as mode = 775 |
|
88
|
|
|
mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH |
|
89
|
|
|
os.chmod(pack_path, mode) |
|
90
|
|
|
|
|
91
|
|
|
# Yuck! Since os.chmod does not support chmod -R walk manually. |
|
92
|
|
|
for root, dirs, files in os.walk(pack_path): |
|
93
|
|
|
for d in dirs: |
|
94
|
|
|
os.chmod(os.path.join(root, d), mode) |
|
95
|
|
|
for f in files: |
|
96
|
|
|
os.chmod(os.path.join(root, f), mode) |
|
97
|
|
|
|
|
98
|
|
|
@staticmethod |
|
99
|
|
|
def _is_desired_pack(abs_pack_path, pack_name): |
|
100
|
|
|
# should not include reserve characters |
|
101
|
|
|
if PACK_RESERVE_CHARACTER in pack_name: |
|
102
|
|
|
return (False, 'Pack name "%s" contains reserve character "%s"' % |
|
103
|
|
|
(pack_name, PACK_RESERVE_CHARACTER)) |
|
104
|
|
|
# must contain a manifest file. Empty file is ok for now. |
|
105
|
|
|
if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)): |
|
106
|
|
|
return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE)) |
|
107
|
|
|
return (True, '') |
|
108
|
|
|
|
|
109
|
|
|
@staticmethod |
|
110
|
|
|
def _cleanup_repo(abs_local_path): |
|
111
|
|
|
# basic lock checking etc? |
|
112
|
|
|
if os.path.isdir(abs_local_path): |
|
113
|
|
|
shutil.rmtree(abs_local_path) |
|
114
|
|
|
|
|
115
|
|
|
@staticmethod |
|
116
|
|
|
def _validate_result(result, repo_url): |
|
117
|
|
|
atleast_one_success = False |
|
118
|
|
|
sanitized_result = {} |
|
119
|
|
|
for k, v in six.iteritems(result): |
|
120
|
|
|
atleast_one_success |= v[0] |
|
121
|
|
|
sanitized_result[k] = v[1] |
|
122
|
|
|
|
|
123
|
|
|
if not atleast_one_success: |
|
124
|
|
|
message_list = [] |
|
125
|
|
|
message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url)) |
|
126
|
|
|
message_list.append('Errors:') |
|
127
|
|
|
|
|
128
|
|
|
for pack, value in result.items(): |
|
129
|
|
|
message_list.append(error) |
|
|
|
|
|
|
130
|
|
|
|
|
131
|
|
|
message = '\n'.join(message_list) |
|
132
|
|
|
raise Exception(message) |
|
133
|
|
|
|
|
134
|
|
|
return sanitized_result |
|
135
|
|
|
|
|
136
|
|
|
def _move_pack(self, abs_repo_base, pack_name, abs_local_path): |
|
137
|
|
|
result = {} |
|
138
|
|
|
|
|
139
|
|
|
desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name) |
|
140
|
|
|
if desired: |
|
141
|
|
|
to = abs_repo_base |
|
142
|
|
|
dest_pack_path = os.path.join(abs_repo_base, pack_name) |
|
143
|
|
|
if os.path.exists(dest_pack_path): |
|
144
|
|
|
self.logger.debug('Removing existing pack %s in %s to replace.', pack_name, |
|
145
|
|
|
dest_pack_path) |
|
146
|
|
|
|
|
147
|
|
|
# Ensure to preserve any existing configuration |
|
148
|
|
|
old_config_file = os.path.join(dest_pack_path, CONFIG_FILE) |
|
149
|
|
|
new_config_file = os.path.join(abs_local_path, CONFIG_FILE) |
|
150
|
|
|
|
|
151
|
|
|
if os.path.isfile(old_config_file): |
|
152
|
|
|
shutil.move(old_config_file, new_config_file) |
|
153
|
|
|
|
|
154
|
|
|
shutil.rmtree(dest_pack_path) |
|
155
|
|
|
|
|
156
|
|
|
self.logger.debug('Moving pack from %s to %s.', abs_local_path, to) |
|
157
|
|
|
shutil.move(abs_local_path, to) |
|
158
|
|
|
# post move fix all permissions. |
|
159
|
|
|
self._apply_pack_permissions(pack_path=dest_pack_path) |
|
160
|
|
|
message = 'Success.' |
|
161
|
|
|
elif message: |
|
162
|
|
|
message = 'Failure : %s' % message |
|
163
|
|
|
|
|
164
|
|
|
result[pack_name] = (desired, message) |
|
165
|
|
|
|
|
166
|
|
|
return result |
|
167
|
|
|
|
|
168
|
|
|
@staticmethod |
|
169
|
|
|
def _get_pack_name_and_url(name_or_url, exchange_url): |
|
170
|
|
|
if len(name_or_url.split('/')) == 1: |
|
171
|
|
|
return (name_or_url, "{}/{}.git".format(exchange_url, name_or_url)) |
|
172
|
|
|
else: |
|
173
|
|
|
return (DownloadGitRepoAction._eval_repo_name(name_or_url), |
|
174
|
|
|
DownloadGitRepoAction._eval_repo_url(name_or_url)) |
|
175
|
|
|
|
|
176
|
|
|
@staticmethod |
|
177
|
|
|
def _eval_repo_url(repo_url): |
|
178
|
|
|
"""Allow passing short GitHub style URLs""" |
|
179
|
|
|
if not repo_url: |
|
180
|
|
|
raise Exception('No valid repo_url provided or could be inferred.') |
|
181
|
|
|
has_git_extension = repo_url.endswith('.git') |
|
182
|
|
|
if len(repo_url.split('/')) == 2 and "git@" not in repo_url: |
|
183
|
|
|
url = "https://github.com/{}".format(repo_url) |
|
184
|
|
|
else: |
|
185
|
|
|
url = repo_url |
|
186
|
|
|
return url if has_git_extension else "{}.git".format(url) |
|
187
|
|
|
|
|
188
|
|
|
@staticmethod |
|
189
|
|
|
def _eval_repo_name(repo_url): |
|
190
|
|
|
""" |
|
191
|
|
|
Evaluate the name of the repo. |
|
192
|
|
|
https://github.com/StackStorm/st2contrib.git -> st2contrib |
|
193
|
|
|
https://github.com/StackStorm/st2contrib -> st2contrib |
|
194
|
|
|
[email protected]:StackStorm/st2contrib.git -> st2contrib |
|
195
|
|
|
[email protected]:StackStorm/st2contrib -> st2contrib |
|
196
|
|
|
""" |
|
197
|
|
|
last_forward_slash = repo_url.rfind('/') |
|
198
|
|
|
next_dot = repo_url.find('.', last_forward_slash) |
|
199
|
|
|
# If dot does not follow last_forward_slash return till the end |
|
200
|
|
|
if next_dot < last_forward_slash: |
|
201
|
|
|
return repo_url[last_forward_slash + 1:] |
|
202
|
|
|
return repo_url[last_forward_slash + 1:next_dot] |
|
203
|
|
|
|
StackStorm /
st2
Loading history...