|
1
|
|
|
# Licensed to the StackStorm, Inc ('StackStorm') under one or more |
|
2
|
|
|
# contributor license agreements. See the NOTICE file distributed with |
|
3
|
|
|
# this work for additional information regarding copyright ownership. |
|
4
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
5
|
|
|
# (the "License"); you may not use this file except in compliance with |
|
6
|
|
|
# the License. You may obtain a copy of the License at |
|
7
|
|
|
# |
|
8
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0 |
|
9
|
|
|
# |
|
10
|
|
|
# Unless required by applicable law or agreed to in writing, software |
|
11
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS, |
|
12
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
13
|
|
|
# See the License for the specific language governing permissions and |
|
14
|
|
|
# limitations under the License. |
|
15
|
|
|
|
|
16
|
|
|
import os |
|
17
|
|
|
import shutil |
|
18
|
|
|
import hashlib |
|
19
|
|
|
import stat |
|
20
|
|
|
import re |
|
21
|
|
|
|
|
22
|
|
|
import six |
|
23
|
|
|
import yaml |
|
24
|
|
|
from git.repo import Repo |
|
25
|
|
|
from lockfile import LockFile |
|
26
|
|
|
|
|
27
|
|
|
from st2common.runners.base_action import Action |
|
28
|
|
|
from st2common.content import utils |
|
29
|
|
|
from st2common.services.packs import get_pack_from_index |
|
30
|
|
|
from st2common.util.green import shell |
|
31
|
|
|
|
|
32
|
|
|
MANIFEST_FILE = 'pack.yaml' |
|
33
|
|
|
CONFIG_FILE = 'config.yaml' |
|
34
|
|
|
GITINFO_FILE = '.gitinfo' |
|
35
|
|
|
PACK_RESERVE_CHARACTER = '.' |
|
36
|
|
|
PACK_VERSION_SEPARATOR = '#' |
|
37
|
|
|
SEMVER_REGEX = "^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[\da-z\-]+(?:\.[\da-z\-]+)*)?(?:\+[\da-z\-]+(?:\.[\da-z\-]+)*)?$" |
|
|
|
|
|
|
38
|
|
|
|
|
39
|
|
|
|
|
40
|
|
|
class DownloadGitRepoAction(Action): |
|
41
|
|
|
def __init__(self, config=None, action_service=None): |
|
42
|
|
|
super(DownloadGitRepoAction, self).__init__(config=config, action_service=action_service) |
|
43
|
|
|
|
|
44
|
|
|
def run(self, packs, abs_repo_base, verifyssl=True): |
|
45
|
|
|
result = {} |
|
46
|
|
|
|
|
47
|
|
|
for pack in packs: |
|
48
|
|
|
pack_url, pack_version = self._get_repo_url(pack) |
|
49
|
|
|
temp_dir = hashlib.md5(pack_url).hexdigest() |
|
50
|
|
|
|
|
51
|
|
|
with LockFile('/tmp/%s' % (temp_dir)): |
|
52
|
|
|
abs_local_path = self._clone_repo(temp_dir=temp_dir, repo_url=pack_url, |
|
53
|
|
|
verifyssl=verifyssl, ref=pack_version) |
|
54
|
|
|
pack_name = self._get_pack_name(abs_local_path) |
|
55
|
|
|
try: |
|
56
|
|
|
result[pack_name] = self._move_pack(abs_repo_base, pack_name, abs_local_path) |
|
57
|
|
|
finally: |
|
58
|
|
|
self._cleanup_repo(abs_local_path) |
|
59
|
|
|
|
|
60
|
|
|
return self._validate_result(result=result, repo_url=pack_url) |
|
61
|
|
|
|
|
62
|
|
|
@staticmethod |
|
63
|
|
|
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref=None): |
|
64
|
|
|
user_home = os.path.expanduser('~') |
|
65
|
|
|
abs_local_path = os.path.join(user_home, temp_dir) |
|
66
|
|
|
|
|
67
|
|
|
# Switch to non-interactive mode |
|
68
|
|
|
os.environ['GIT_TERMINAL_PROMPT'] = '0' |
|
69
|
|
|
|
|
70
|
|
|
# Disable SSL cert checking if explictly asked |
|
71
|
|
|
if not verifyssl: |
|
72
|
|
|
os.environ['GIT_SSL_NO_VERIFY'] = 'true' |
|
73
|
|
|
|
|
74
|
|
|
if not ref: |
|
75
|
|
|
ref = 'master' |
|
76
|
|
|
|
|
77
|
|
|
# Clone the repo from git; we don't use shallow copying |
|
78
|
|
|
# because we want the user to work with the repo in the |
|
79
|
|
|
# future. |
|
80
|
|
|
repo = Repo.clone_from(repo_url, abs_local_path) |
|
81
|
|
|
|
|
82
|
|
|
if not repo.commit(ref): |
|
83
|
|
|
if re.match(SEMVER_REGEX, ref) and repo.commit("v%s" % ref): |
|
84
|
|
|
ref = "v%s" % ref |
|
85
|
|
|
else: |
|
86
|
|
|
raise ValueError("\"%s\" is not a valid ref in %s." % (ref, repo_url)) |
|
87
|
|
|
|
|
88
|
|
|
repo.head.reference = repo.commit(ref) |
|
89
|
|
|
repo.head.reset(index=True, working_tree=True) |
|
90
|
|
|
|
|
91
|
|
|
return abs_local_path |
|
92
|
|
|
|
|
93
|
|
|
def _move_pack(self, abs_repo_base, pack_name, abs_local_path): |
|
94
|
|
|
desired, message = DownloadGitRepoAction._is_desired_pack(abs_local_path, pack_name) |
|
95
|
|
|
if desired: |
|
96
|
|
|
to = abs_repo_base |
|
97
|
|
|
dest_pack_path = os.path.join(abs_repo_base, pack_name) |
|
98
|
|
|
if os.path.exists(dest_pack_path): |
|
99
|
|
|
self.logger.debug('Removing existing pack %s in %s to replace.', pack_name, |
|
100
|
|
|
dest_pack_path) |
|
101
|
|
|
|
|
102
|
|
|
# Ensure to preserve any existing configuration |
|
103
|
|
|
old_config_file = os.path.join(dest_pack_path, CONFIG_FILE) |
|
104
|
|
|
new_config_file = os.path.join(abs_local_path, CONFIG_FILE) |
|
105
|
|
|
|
|
106
|
|
|
if os.path.isfile(old_config_file): |
|
107
|
|
|
shutil.move(old_config_file, new_config_file) |
|
108
|
|
|
|
|
109
|
|
|
shutil.rmtree(dest_pack_path) |
|
110
|
|
|
|
|
111
|
|
|
self.logger.debug('Moving pack from %s to %s.', abs_local_path, to) |
|
112
|
|
|
shutil.move(abs_local_path, dest_pack_path) |
|
113
|
|
|
# post move fix all permissions. |
|
114
|
|
|
self._apply_pack_permissions(pack_path=dest_pack_path) |
|
115
|
|
|
message = 'Success.' |
|
116
|
|
|
elif message: |
|
117
|
|
|
message = 'Failure : %s' % message |
|
118
|
|
|
|
|
119
|
|
|
return (desired, message) |
|
120
|
|
|
|
|
121
|
|
|
def _apply_pack_permissions(self, pack_path): |
|
122
|
|
|
""" |
|
123
|
|
|
Will recursively apply permission 770 to pack and its contents. |
|
124
|
|
|
""" |
|
125
|
|
|
# 1. switch owner group to configured group |
|
126
|
|
|
pack_group = utils.get_pack_group() |
|
127
|
|
|
if pack_group: |
|
128
|
|
|
shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path]) |
|
129
|
|
|
|
|
130
|
|
|
# 2. Setup the right permissions and group ownership |
|
131
|
|
|
# These mask is same as mode = 775 |
|
132
|
|
|
mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH |
|
133
|
|
|
os.chmod(pack_path, mode) |
|
134
|
|
|
|
|
135
|
|
|
# Yuck! Since os.chmod does not support chmod -R walk manually. |
|
136
|
|
|
for root, dirs, files in os.walk(pack_path): |
|
137
|
|
|
for d in dirs: |
|
138
|
|
|
os.chmod(os.path.join(root, d), mode) |
|
139
|
|
|
for f in files: |
|
140
|
|
|
os.chmod(os.path.join(root, f), mode) |
|
141
|
|
|
|
|
142
|
|
|
@staticmethod |
|
143
|
|
|
def _is_desired_pack(abs_pack_path, pack_name): |
|
144
|
|
|
# path has to exist. |
|
145
|
|
|
if not os.path.exists(abs_pack_path): |
|
146
|
|
|
return (False, 'Pack "%s" not found or it\'s missing a "pack.yaml" file.' % |
|
147
|
|
|
(pack_name)) |
|
148
|
|
|
# should not include reserve characters |
|
149
|
|
|
if PACK_RESERVE_CHARACTER in pack_name: |
|
150
|
|
|
return (False, 'Pack name "%s" contains reserve character "%s"' % |
|
151
|
|
|
(pack_name, PACK_RESERVE_CHARACTER)) |
|
152
|
|
|
# must contain a manifest file. Empty file is ok for now. |
|
153
|
|
|
if not os.path.isfile(os.path.join(abs_pack_path, MANIFEST_FILE)): |
|
154
|
|
|
return (False, 'Pack is missing a manifest file (%s).' % (MANIFEST_FILE)) |
|
155
|
|
|
return (True, '') |
|
156
|
|
|
|
|
157
|
|
|
@staticmethod |
|
158
|
|
|
def _cleanup_repo(abs_local_path): |
|
159
|
|
|
# basic lock checking etc? |
|
160
|
|
|
if os.path.isdir(abs_local_path): |
|
161
|
|
|
shutil.rmtree(abs_local_path) |
|
162
|
|
|
|
|
163
|
|
|
@staticmethod |
|
164
|
|
|
def _validate_result(result, repo_url): |
|
165
|
|
|
atleast_one_success = False |
|
166
|
|
|
sanitized_result = {} |
|
167
|
|
|
for k, v in six.iteritems(result): |
|
168
|
|
|
atleast_one_success |= v[0] |
|
169
|
|
|
sanitized_result[k] = v[1] |
|
170
|
|
|
|
|
171
|
|
|
if not atleast_one_success: |
|
172
|
|
|
message_list = [] |
|
173
|
|
|
message_list.append('The pack has not been downloaded from "%s".\n' % (repo_url)) |
|
174
|
|
|
message_list.append('Errors:') |
|
175
|
|
|
|
|
176
|
|
|
for pack, value in result.items(): |
|
177
|
|
|
success, error = value |
|
178
|
|
|
message_list.append(error) |
|
179
|
|
|
|
|
180
|
|
|
message = '\n'.join(message_list) |
|
181
|
|
|
raise Exception(message) |
|
182
|
|
|
|
|
183
|
|
|
return sanitized_result |
|
184
|
|
|
|
|
185
|
|
|
@staticmethod |
|
186
|
|
|
def _get_repo_url(pack): |
|
187
|
|
|
pack_and_version = pack.split(PACK_VERSION_SEPARATOR) |
|
188
|
|
|
name_or_url = pack_and_version[0] |
|
189
|
|
|
version = pack_and_version[1] if len(pack_and_version) > 1 else None |
|
190
|
|
|
|
|
191
|
|
|
if len(name_or_url.split('/')) == 1: |
|
192
|
|
|
pack = get_pack_from_index(name_or_url) |
|
193
|
|
|
if not pack: |
|
194
|
|
|
raise Exception('No record of the "%s" pack in the index.' % name_or_url) |
|
195
|
|
|
return (pack['repo_url'], version) |
|
196
|
|
|
else: |
|
197
|
|
|
return (DownloadGitRepoAction._eval_repo_url(name_or_url), version) |
|
198
|
|
|
|
|
199
|
|
|
@staticmethod |
|
200
|
|
|
def _eval_repo_url(repo_url): |
|
201
|
|
|
"""Allow passing short GitHub style URLs""" |
|
202
|
|
|
if not repo_url: |
|
203
|
|
|
raise Exception('No valid repo_url provided or could be inferred.') |
|
204
|
|
|
has_git_extension = repo_url.endswith('.git') |
|
205
|
|
|
if len(repo_url.split('/')) == 2 and "git@" not in repo_url: |
|
206
|
|
|
url = "https://github.com/{}".format(repo_url) |
|
207
|
|
|
else: |
|
208
|
|
|
url = repo_url |
|
209
|
|
|
return url if has_git_extension else "{}.git".format(url) |
|
210
|
|
|
|
|
211
|
|
|
@staticmethod |
|
212
|
|
|
def _get_pack_name(pack_dir): |
|
213
|
|
|
""" |
|
214
|
|
|
Read pack name from the metadata file and sanitize it. |
|
215
|
|
|
""" |
|
216
|
|
|
with open(os.path.join(pack_dir, MANIFEST_FILE), 'r') as manifest_file: |
|
217
|
|
|
pack_meta = yaml.load(manifest_file) |
|
218
|
|
|
return pack_meta['name'].replace(' ', '-').lower() |
|
219
|
|
|
|
StackStorm /
st2
Loading history...
This check looks for lines that are too long. You can specify the maximum line length.