|
1
|
|
|
# Licensed to the StackStorm, Inc ('StackStorm') under one or more |
|
2
|
|
|
# contributor license agreements. See the NOTICE file distributed with |
|
3
|
|
|
# this work for additional information regarding copyright ownership. |
|
4
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0 |
|
5
|
|
|
# (the "License"); you may not use this file except in compliance with |
|
6
|
|
|
# the License. You may obtain a copy of the License at |
|
7
|
|
|
# |
|
8
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0 |
|
9
|
|
|
# |
|
10
|
|
|
# Unless required by applicable law or agreed to in writing, software |
|
11
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS, |
|
12
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
13
|
|
|
# See the License for the specific language governing permissions and |
|
14
|
|
|
# limitations under the License. |
|
15
|
|
|
|
|
16
|
|
|
import copy |
|
17
|
|
|
import json |
|
18
|
|
|
import re |
|
19
|
|
|
|
|
20
|
|
|
import requests |
|
21
|
|
|
import six |
|
22
|
|
|
import yaml |
|
23
|
|
|
from mistralclient.api.base import APIException |
|
24
|
|
|
|
|
25
|
|
|
from st2common.exceptions.workflow import WorkflowDefinitionException |
|
26
|
|
|
from st2common import log as logging |
|
27
|
|
|
from st2common.models.system.common import ResourceReference |
|
28
|
|
|
from st2common.models.utils import action_param_utils |
|
29
|
|
|
from st2common.util import action_db as action_utils |
|
30
|
|
|
|
|
31
|
|
|
|
|
32
|
|
|
LOG = logging.getLogger(__name__) |
|
33
|
|
|
|
|
34
|
|
|
|
|
35
|
|
|
CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*") |
|
|
|
|
|
|
36
|
|
|
|
|
37
|
|
|
INLINE_YAQL = '<%.*?%>' |
|
38
|
|
|
_ALL_IN_BRACKETS = "\[.*\]\s*" |
|
|
|
|
|
|
39
|
|
|
_ALL_IN_QUOTES = "\"[^\"]*\"\s*" |
|
|
|
|
|
|
40
|
|
|
_ALL_IN_APOSTROPHES = "'[^']*'\s*" |
|
|
|
|
|
|
41
|
|
|
_DIGITS = "\d+" |
|
|
|
|
|
|
42
|
|
|
_TRUE = "true" |
|
43
|
|
|
_FALSE = "false" |
|
44
|
|
|
_NULL = "null" |
|
45
|
|
|
|
|
46
|
|
|
ALL = ( |
|
47
|
|
|
_ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL, |
|
48
|
|
|
_ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS |
|
49
|
|
|
) |
|
50
|
|
|
|
|
51
|
|
|
PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "|".join(ALL)) |
|
|
|
|
|
|
52
|
|
|
|
|
53
|
|
|
SPEC_TYPES = { |
|
54
|
|
|
'adhoc': { |
|
55
|
|
|
'action_key': 'base', |
|
56
|
|
|
'input_key': 'base-input' |
|
57
|
|
|
}, |
|
58
|
|
|
'task': { |
|
59
|
|
|
'action_key': 'action', |
|
60
|
|
|
'input_key': 'input' |
|
61
|
|
|
} |
|
62
|
|
|
} |
|
63
|
|
|
|
|
64
|
|
|
JINJA_WITH_ST2KV = '{{st2kv\..*?|.*?\sst2kv\..*?}}' |
|
|
|
|
|
|
65
|
|
|
JINJA_WITH_ST2KV_PTRN = re.compile(JINJA_WITH_ST2KV) |
|
66
|
|
|
JINJA_WITH_LOCAL_CTX = '{{.*?_\..*?}}' |
|
|
|
|
|
|
67
|
|
|
JINJA_WITH_LOCAL_CTX_PTRN = re.compile(JINJA_WITH_LOCAL_CTX) |
|
68
|
|
|
|
|
69
|
|
|
|
|
70
|
|
|
def _parse_cmd_and_input(cmd_str): |
|
71
|
|
|
cmd_matcher = CMD_PTRN.search(cmd_str) |
|
72
|
|
|
|
|
73
|
|
|
if not cmd_matcher: |
|
74
|
|
|
raise ValueError("Invalid action/workflow task property: %s" % cmd_str) |
|
75
|
|
|
|
|
76
|
|
|
cmd = cmd_matcher.group() |
|
77
|
|
|
|
|
78
|
|
|
params = {} |
|
79
|
|
|
for k, v in re.findall(PARAMS_PTRN, cmd_str): |
|
80
|
|
|
# Remove embracing quotes. |
|
81
|
|
|
v = v.strip() |
|
82
|
|
|
if v[0] == '"' or v[0] == "'": |
|
83
|
|
|
v = v[1:-1] |
|
84
|
|
|
else: |
|
85
|
|
|
try: |
|
86
|
|
|
v = json.loads(v) |
|
87
|
|
|
except Exception: |
|
88
|
|
|
pass |
|
89
|
|
|
|
|
90
|
|
|
params[k] = v |
|
91
|
|
|
|
|
92
|
|
|
return cmd, params |
|
93
|
|
|
|
|
94
|
|
|
|
|
95
|
|
|
def _merge_dicts(left, right): |
|
96
|
|
|
if left is None: |
|
97
|
|
|
return right |
|
98
|
|
|
|
|
99
|
|
|
if right is None: |
|
100
|
|
|
return left |
|
101
|
|
|
|
|
102
|
|
|
for k, v in right.iteritems(): |
|
103
|
|
|
if k not in left: |
|
104
|
|
|
left[k] = v |
|
105
|
|
|
else: |
|
106
|
|
|
left_v = left[k] |
|
107
|
|
|
|
|
108
|
|
|
if isinstance(left_v, dict) and isinstance(v, dict): |
|
109
|
|
|
_merge_dicts(left_v, v) |
|
110
|
|
|
|
|
111
|
|
|
return left |
|
112
|
|
|
|
|
113
|
|
|
|
|
114
|
|
|
def _eval_inline_params(spec, action_key, input_key): |
|
115
|
|
|
action_str = spec.get(action_key) |
|
116
|
|
|
command, inputs = _parse_cmd_and_input(action_str) |
|
117
|
|
|
if inputs: |
|
118
|
|
|
spec[action_key] = command |
|
119
|
|
|
if input_key not in spec: |
|
120
|
|
|
spec[input_key] = {} |
|
121
|
|
|
_merge_dicts(spec[input_key], inputs) |
|
122
|
|
|
|
|
123
|
|
|
|
|
124
|
|
|
def _validate_action_parameters(name, action, action_params): |
|
125
|
|
|
requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params) |
|
126
|
|
|
|
|
127
|
|
|
if requires: |
|
128
|
|
|
raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": ' |
|
129
|
|
|
'"%s"' % (name, action.ref, '", "'.join(requires))) |
|
130
|
|
|
|
|
131
|
|
|
if unexpected: |
|
132
|
|
|
raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": ' |
|
133
|
|
|
'"%s"' % (name, action.ref, '", "'.join(unexpected))) |
|
134
|
|
|
|
|
135
|
|
|
|
|
136
|
|
|
def _transform_action_param(action_ref, param_name, param_value): |
|
137
|
|
|
if isinstance(param_value, list): |
|
138
|
|
|
param_value = [ |
|
139
|
|
|
_transform_action_param(action_ref, param_name, value) |
|
140
|
|
|
for value in param_value |
|
141
|
|
|
] |
|
142
|
|
|
|
|
143
|
|
|
if isinstance(param_value, dict): |
|
144
|
|
|
param_value = { |
|
145
|
|
|
name: _transform_action_param(action_ref, name, value) |
|
146
|
|
|
for name, value in six.iteritems(param_value) |
|
147
|
|
|
} |
|
148
|
|
|
|
|
149
|
|
|
if isinstance(param_value, six.string_types): |
|
150
|
|
|
st2kv_matches = JINJA_WITH_ST2KV_PTRN.findall(param_value) |
|
151
|
|
|
local_ctx_matches = JINJA_WITH_LOCAL_CTX_PTRN.findall(param_value) |
|
152
|
|
|
|
|
153
|
|
|
if st2kv_matches and local_ctx_matches: |
|
154
|
|
|
raise WorkflowDefinitionException('Parameter "%s" for action "%s" containing ' |
|
155
|
|
|
'references to both local context (i.e. _.var1) ' |
|
156
|
|
|
'and st2kv (i.e. st2kv.system.var1) is not ' |
|
157
|
|
|
'supported.' % (param_name, action_ref)) |
|
158
|
|
|
|
|
159
|
|
|
if st2kv_matches: |
|
160
|
|
|
param_value = '{% raw %}' + param_value + '{% endraw %}' |
|
161
|
|
|
|
|
162
|
|
|
return param_value |
|
163
|
|
|
|
|
164
|
|
|
|
|
165
|
|
|
def _transform_action(name, spec): |
|
166
|
|
|
|
|
167
|
|
|
action_key, input_key = None, None |
|
168
|
|
|
|
|
169
|
|
|
for spec_type, spec_meta in six.iteritems(SPEC_TYPES): |
|
170
|
|
|
if spec_meta['action_key'] in spec: |
|
171
|
|
|
action_key = spec_meta['action_key'] |
|
172
|
|
|
input_key = spec_meta['input_key'] |
|
173
|
|
|
break |
|
174
|
|
|
|
|
175
|
|
|
if not action_key: |
|
176
|
|
|
return |
|
177
|
|
|
|
|
178
|
|
|
if spec[action_key] == 'st2.callback': |
|
179
|
|
|
raise WorkflowDefinitionException('st2.callback is deprecated.') |
|
180
|
|
|
|
|
181
|
|
|
# Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2}) |
|
182
|
|
|
# and split it to action name and input dict as illustrated below. |
|
183
|
|
|
# |
|
184
|
|
|
# action: some_action |
|
185
|
|
|
# input: |
|
186
|
|
|
# var1: <% $.value1 %> |
|
187
|
|
|
# var2: <% $.value2 %> |
|
188
|
|
|
# |
|
189
|
|
|
# This step to separate the action name and the input parameters is required |
|
190
|
|
|
# to wrap them with the st2.action proxy. |
|
191
|
|
|
# |
|
192
|
|
|
# action: st2.action |
|
193
|
|
|
# input: |
|
194
|
|
|
# ref: some_action |
|
195
|
|
|
# parameters: |
|
196
|
|
|
# var1: <% $.value1 %> |
|
197
|
|
|
# var2: <% $.value2 %> |
|
198
|
|
|
_eval_inline_params(spec, action_key, input_key) |
|
199
|
|
|
|
|
200
|
|
|
transformed = (spec[action_key] == 'st2.action') |
|
201
|
|
|
|
|
202
|
|
|
action_ref = spec[input_key]['ref'] if transformed else spec[action_key] |
|
203
|
|
|
|
|
204
|
|
|
action = None |
|
205
|
|
|
|
|
206
|
|
|
# Identify if action is a registered StackStorm action. |
|
207
|
|
|
if action_ref and ResourceReference.is_resource_reference(action_ref): |
|
208
|
|
|
action = action_utils.get_action_by_ref(ref=action_ref) |
|
209
|
|
|
|
|
210
|
|
|
# If action is a registered StackStorm action, then wrap the |
|
211
|
|
|
# action with the st2 proxy and validate the action input. |
|
212
|
|
|
if action: |
|
213
|
|
|
if not transformed: |
|
214
|
|
|
spec[action_key] = 'st2.action' |
|
215
|
|
|
action_input = spec.get(input_key) |
|
216
|
|
|
spec[input_key] = {'ref': action_ref} |
|
217
|
|
|
if action_input: |
|
218
|
|
|
spec[input_key]['parameters'] = action_input |
|
219
|
|
|
|
|
220
|
|
|
action_input = spec.get(input_key, {}) |
|
221
|
|
|
action_params = action_input.get('parameters', {}) |
|
222
|
|
|
_validate_action_parameters(name, action, action_params) |
|
223
|
|
|
|
|
224
|
|
|
xformed_action_params = {} |
|
225
|
|
|
|
|
226
|
|
|
for param_name in action_params.keys(): |
|
227
|
|
|
param_value = copy.deepcopy(action_params[param_name]) |
|
228
|
|
|
xformed_param_value = _transform_action_param( |
|
229
|
|
|
action_ref, param_name, param_value) |
|
230
|
|
|
xformed_action_params[param_name] = xformed_param_value |
|
231
|
|
|
|
|
232
|
|
|
if xformed_action_params != action_params: |
|
233
|
|
|
spec[input_key]['parameters'] = xformed_action_params |
|
234
|
|
|
|
|
235
|
|
|
|
|
236
|
|
|
def transform_definition(definition): |
|
237
|
|
|
# If definition is a dictionary, there is no need to load from YAML. |
|
238
|
|
|
is_dict = isinstance(definition, dict) |
|
239
|
|
|
spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition) |
|
240
|
|
|
|
|
241
|
|
|
# Transform adhoc actions |
|
242
|
|
|
for action_name, action_spec in six.iteritems(spec.get('actions', {})): |
|
243
|
|
|
_transform_action(action_name, action_spec) |
|
244
|
|
|
|
|
245
|
|
|
# Determine if definition is a workbook or workflow |
|
246
|
|
|
is_workbook = 'workflows' in spec |
|
247
|
|
|
|
|
248
|
|
|
# Transform tasks |
|
249
|
|
|
if is_workbook: |
|
250
|
|
|
for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})): |
|
251
|
|
|
if 'tasks' in workflow_spec: |
|
252
|
|
|
for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')): |
|
253
|
|
|
_transform_action(task_name, task_spec) |
|
254
|
|
|
else: |
|
255
|
|
|
for key, value in six.iteritems(spec): |
|
256
|
|
|
if 'tasks' in value: |
|
257
|
|
|
for task_name, task_spec in six.iteritems(value.get('tasks')): |
|
258
|
|
|
_transform_action(task_name, task_spec) |
|
259
|
|
|
|
|
260
|
|
|
# Return the same type as original input. |
|
261
|
|
|
return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False) |
|
262
|
|
|
|
|
263
|
|
|
|
|
264
|
|
|
def retry_on_exceptions(exc): |
|
265
|
|
|
LOG.warning('Determining if %s should be retried...', type(exc)) |
|
266
|
|
|
|
|
267
|
|
|
is_connection_error = isinstance(exc, requests.exceptions.ConnectionError) |
|
268
|
|
|
is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message |
|
269
|
|
|
is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message |
|
270
|
|
|
retrying = is_connection_error or is_duplicate_error or is_messaging_error |
|
271
|
|
|
|
|
272
|
|
|
if retrying: |
|
273
|
|
|
LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc)) |
|
274
|
|
|
|
|
275
|
|
|
return retrying |
|
276
|
|
|
|
StackStorm /
st2
Loading history...
Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with
rorRare they interpreted as regular expressions.The escape sequence that was used indicates that you might have intended to write a regular expression.
Learn more about the available escape sequences. in the Python documentation.