_transform_action_param() - Code Metrics - Inspection of "Fix mistral upstream integration" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#3154)

by W

created 2017-01-10 01:00 UTC

_transform_action_param() A

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	2
c	0
b	0
f	0
dl	0
loc	7
rs	9.4285

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import json
import re

import requests
import six
import yaml
from mistralclient.api.base import APIException

from st2common.exceptions.workflow import WorkflowDefinitionException
from st2common import log as logging
from st2common.models.system.common import ResourceReference
from st2common.models.utils import action_param_utils
from st2common.util import action_db as action_utils


LOG = logging.getLogger(__name__)


CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*")


INLINE_YAQL = '<%.*?%>'
_ALL_IN_BRACKETS = "\[.*\]\s*"

_ALL_IN_QUOTES = "\"[^\"]*\"\s*"

_ALL_IN_APOSTROPHES = "'[^']*'\s*"

_DIGITS = "\d+"

_TRUE = "true"
_FALSE = "false"
_NULL = "null"

ALL = (
    _ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL,
    _ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS
)

PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "|".join(ALL))


SPEC_TYPES = {
    'adhoc': {
        'action_key': 'base',
        'input_key': 'base-input'
    },
    'task': {
        'action_key': 'action',
        'input_key': 'input'
    }
}

JINJA_WITH_ST2KV = '{{.*?st2kv\..*?}}'

JINJA_WITH_ST2KV_PTRN = re.compile(JINJA_WITH_ST2KV)


def _parse_cmd_and_input(cmd_str):
    cmd_matcher = CMD_PTRN.search(cmd_str)

    if not cmd_matcher:
        raise ValueError("Invalid action/workflow task property: %s" % cmd_str)

    cmd = cmd_matcher.group()

    params = {}
    for k, v in re.findall(PARAMS_PTRN, cmd_str):
        # Remove embracing quotes.
        v = v.strip()
        if v[0] == '"' or v[0] == "'":
            v = v[1:-1]
        else:
            try:
                v = json.loads(v)
            except Exception:
                pass

        params[k] = v

    return cmd, params


def _merge_dicts(left, right):
    if left is None:
        return right

    if right is None:
        return left

    for k, v in right.iteritems():
        if k not in left:
            left[k] = v
        else:
            left_v = left[k]

            if isinstance(left_v, dict) and isinstance(v, dict):
                _merge_dicts(left_v, v)

    return left


def _eval_inline_params(spec, action_key, input_key):
    action_str = spec.get(action_key)
    command, inputs = _parse_cmd_and_input(action_str)
    if inputs:
        spec[action_key] = command
        if input_key not in spec:
            spec[input_key] = {}
            _merge_dicts(spec[input_key], inputs)


def _validate_action_parameters(name, action, action_params):
    requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params)

    if requires:
        raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": '
                                          '"%s"' % (name, action.ref, '", "'.join(requires)))

    if unexpected:
        raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": '
                                          '"%s"' % (name, action.ref, '", "'.join(unexpected)))


def _transform_action_param(value):
    matches = JINJA_WITH_ST2KV_PTRN.findall(value)

    if matches:
        value = '{% raw %}' + value + '{% endraw %}'

    return value


def _transform_action(name, spec):

    action_key, input_key = None, None

    for spec_type, spec_meta in six.iteritems(SPEC_TYPES):
        if spec_meta['action_key'] in spec:
            action_key = spec_meta['action_key']
            input_key = spec_meta['input_key']
            break

    if not action_key:
        return

    if spec[action_key] == 'st2.callback':
        raise WorkflowDefinitionException('st2.callback is deprecated.')

    # Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2})
    # and split it to action name and input dict as illustrated below.
    #
    # action: some_action
    # input:
    #   var1: <% $.value1 %>
    #   var2: <% $.value2 %>
    #
    # This step to separate the action name and the input parameters is required
    # to wrap them with the st2.action proxy.
    #
    # action: st2.action
    # input:
    #   ref: some_action
    #   parameters:
    #     var1: <% $.value1 %>
    #     var2: <% $.value2 %>
    _eval_inline_params(spec, action_key, input_key)

    transformed = (spec[action_key] == 'st2.action')

    action_ref = spec[input_key]['ref'] if transformed else spec[action_key]

    action = None

    # Identify if action is a registered StackStorm action.
    if action_ref and ResourceReference.is_resource_reference(action_ref):
        action = action_utils.get_action_by_ref(ref=action_ref)

    # If action is a registered StackStorm action, then wrap the
    # action with the st2 proxy and validate the action input.
    if action:
        if not transformed:
            spec[action_key] = 'st2.action'
            action_input = spec.get(input_key)
            spec[input_key] = {'ref': action_ref}
            if action_input:
                spec[input_key]['parameters'] = action_input

        action_input = spec.get(input_key, {})
        action_params = action_input.get('parameters', {})
        _validate_action_parameters(name, action, action_params)

        xformed_action_params = {}

        for param_name in action_params.keys():
            param_value = copy.deepcopy(action_params[param_name])
            xformed_param_value = _transform_action_param(param_value)
            xformed_action_params[param_name] = xformed_param_value

        if xformed_action_params != action_params:
            spec[input_key]['parameters'] = xformed_action_params


def transform_definition(definition):
    # If definition is a dictionary, there is no need to load from YAML.
    is_dict = isinstance(definition, dict)
    spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition)

    # Transform adhoc actions
    for action_name, action_spec in six.iteritems(spec.get('actions', {})):
        _transform_action(action_name, action_spec)

    # Determine if definition is a workbook or workflow
    is_workbook = 'workflows' in spec

    # Transform tasks
    if is_workbook:
        for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})):
            if 'tasks' in workflow_spec:
                for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')):
                    _transform_action(task_name, task_spec)
    else:
        for key, value in six.iteritems(spec):
            if 'tasks' in value:
                for task_name, task_spec in six.iteritems(value.get('tasks')):
                    _transform_action(task_name, task_spec)

    # Return the same type as original input.
    return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False)


def retry_on_exceptions(exc):
    LOG.warning('Determining if %s should be retried...', type(exc))

    is_connection_error = isinstance(exc, requests.exceptions.ConnectionError)
    is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message
    is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message
    retrying = is_connection_error or is_duplicate_error or is_messaging_error

    if retrying:
        LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc))

    return retrying


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import copy
17			import json
18			import re
19
20			import requests
21			import six
22			import yaml
23			from mistralclient.api.base import APIException
24
25			from st2common.exceptions.workflow import WorkflowDefinitionException
26			from st2common import log as logging
27			from st2common.models.system.common import ResourceReference
28			from st2common.models.utils import action_param_utils
29			from st2common.util import action_db as action_utils
30
31
32			LOG = logging.getLogger(__name__)
33
34
35			CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*")
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\w` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\.` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
36
37			INLINE_YAQL = '<%.*?%>'
38			_ALL_IN_BRACKETS = "\[.\]\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\[` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\]` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
39			_ALL_IN_QUOTES = "\"[^\"]\"\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
40			_ALL_IN_APOSTROPHES = "'[^']'\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
41			_DIGITS = "\d+"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\d` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
42			_TRUE = "true"
43			_FALSE = "false"
44			_NULL = "null"
45
46			ALL = (
47			_ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL,
48			_ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS
49			)
50
51			PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "\|".join(ALL))
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\w` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
52
53			SPEC_TYPES = {
54			'adhoc': {
55			'action_key': 'base',
56			'input_key': 'base-input'
57			},
58			'task': {
59			'action_key': 'action',
60			'input_key': 'input'
61			}
62			}
63
64			JINJA_WITH_ST2KV = '{{.?st2kv\..?}}'
			0 ignored issues – show Bug introduced 2017-01-10 01:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\.` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
65			JINJA_WITH_ST2KV_PTRN = re.compile(JINJA_WITH_ST2KV)
66
67
68			def _parse_cmd_and_input(cmd_str):
69			cmd_matcher = CMD_PTRN.search(cmd_str)
70
71			if not cmd_matcher:
72			raise ValueError("Invalid action/workflow task property: %s" % cmd_str)
73
74			cmd = cmd_matcher.group()
75
76			params = {}
77			for k, v in re.findall(PARAMS_PTRN, cmd_str):
78			# Remove embracing quotes.
79			v = v.strip()
80			if v[0] == '"' or v[0] == "'":
81			v = v[1:-1]
82			else:
83			try:
84			v = json.loads(v)
85			except Exception:
86			pass
87
88			params[k] = v
89
90			return cmd, params
91
92
93			def _merge_dicts(left, right):
94			if left is None:
95			return right
96
97			if right is None:
98			return left
99
100			for k, v in right.iteritems():
101			if k not in left:
102			left[k] = v
103			else:
104			left_v = left[k]
105
106			if isinstance(left_v, dict) and isinstance(v, dict):
107			_merge_dicts(left_v, v)
108
109			return left
110
111
112			def _eval_inline_params(spec, action_key, input_key):
113			action_str = spec.get(action_key)
114			command, inputs = _parse_cmd_and_input(action_str)
115			if inputs:
116			spec[action_key] = command
117			if input_key not in spec:
118			spec[input_key] = {}
119			_merge_dicts(spec[input_key], inputs)
120
121
122			def _validate_action_parameters(name, action, action_params):
123			requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params)
124
125			if requires:
126			raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": '
127			'"%s"' % (name, action.ref, '", "'.join(requires)))
128
129			if unexpected:
130			raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": '
131			'"%s"' % (name, action.ref, '", "'.join(unexpected)))
132
133
134			def _transform_action_param(value):
135			matches = JINJA_WITH_ST2KV_PTRN.findall(value)
136
137			if matches:
138			value = '{% raw %}' + value + '{% endraw %}'
139
140			return value
141
142
143			def _transform_action(name, spec):
144
145			action_key, input_key = None, None
146
147			for spec_type, spec_meta in six.iteritems(SPEC_TYPES):
148			if spec_meta['action_key'] in spec:
149			action_key = spec_meta['action_key']
150			input_key = spec_meta['input_key']
151			break
152
153			if not action_key:
154			return
155
156			if spec[action_key] == 'st2.callback':
157			raise WorkflowDefinitionException('st2.callback is deprecated.')
158
159			# Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2})
160			# and split it to action name and input dict as illustrated below.
161			#
162			# action: some_action
163			# input:
164			# var1: <% $.value1 %>
165			# var2: <% $.value2 %>
166			#
167			# This step to separate the action name and the input parameters is required
168			# to wrap them with the st2.action proxy.
169			#
170			# action: st2.action
171			# input:
172			# ref: some_action
173			# parameters:
174			# var1: <% $.value1 %>
175			# var2: <% $.value2 %>
176			_eval_inline_params(spec, action_key, input_key)
177
178			transformed = (spec[action_key] == 'st2.action')
179
180			action_ref = spec[input_key]['ref'] if transformed else spec[action_key]
181
182			action = None
183
184			# Identify if action is a registered StackStorm action.
185			if action_ref and ResourceReference.is_resource_reference(action_ref):
186			action = action_utils.get_action_by_ref(ref=action_ref)
187
188			# If action is a registered StackStorm action, then wrap the
189			# action with the st2 proxy and validate the action input.
190			if action:
191			if not transformed:
192			spec[action_key] = 'st2.action'
193			action_input = spec.get(input_key)
194			spec[input_key] = {'ref': action_ref}
195			if action_input:
196			spec[input_key]['parameters'] = action_input
197
198			action_input = spec.get(input_key, {})
199			action_params = action_input.get('parameters', {})
200			_validate_action_parameters(name, action, action_params)
201
202			xformed_action_params = {}
203
204			for param_name in action_params.keys():
205			param_value = copy.deepcopy(action_params[param_name])
206			xformed_param_value = _transform_action_param(param_value)
207			xformed_action_params[param_name] = xformed_param_value
208
209			if xformed_action_params != action_params:
210			spec[input_key]['parameters'] = xformed_action_params
211
212
213			def transform_definition(definition):
214			# If definition is a dictionary, there is no need to load from YAML.
215			is_dict = isinstance(definition, dict)
216			spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition)
217
218			# Transform adhoc actions
219			for action_name, action_spec in six.iteritems(spec.get('actions', {})):
220			_transform_action(action_name, action_spec)
221
222			# Determine if definition is a workbook or workflow
223			is_workbook = 'workflows' in spec
224
225			# Transform tasks
226			if is_workbook:
227			for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})):
228			if 'tasks' in workflow_spec:
229			for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')):
230			_transform_action(task_name, task_spec)
231			else:
232			for key, value in six.iteritems(spec):
233			if 'tasks' in value:
234			for task_name, task_spec in six.iteritems(value.get('tasks')):
235			_transform_action(task_name, task_spec)
236
237			# Return the same type as original input.
238			return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False)
239
240
241			def retry_on_exceptions(exc):
242			LOG.warning('Determining if %s should be retried...', type(exc))
243
244			is_connection_error = isinstance(exc, requests.exceptions.ConnectionError)
245			is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message
246			is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message
247			retrying = is_connection_error or is_duplicate_error or is_messaging_error
248
249			if retrying:
250			LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc))
251
252			return retrying
253

StackStorm / st2

Pull Request — master (#3154)

_transform_action_param() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like