_transform_action() - Code Metrics - Inspection of "Fix mistral upstream integration" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#3154)

by W

created 2017-01-10 21:45 UTC

_transform_action() F

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	13
dl	0
loc	69
rs	2.4914
c	0
b	0
f	0

How to fix Long Method Complexity

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import json
import re

import requests
import six
import yaml
from mistralclient.api.base import APIException

from st2common.exceptions.workflow import WorkflowDefinitionException
from st2common import log as logging
from st2common.models.system.common import ResourceReference
from st2common.models.utils import action_param_utils
from st2common.util import action_db as action_utils


LOG = logging.getLogger(__name__)


CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*")


INLINE_YAQL = '<%.*?%>'
_ALL_IN_BRACKETS = "\[.*\]\s*"

_ALL_IN_QUOTES = "\"[^\"]*\"\s*"

_ALL_IN_APOSTROPHES = "'[^']*'\s*"

_DIGITS = "\d+"

_TRUE = "true"
_FALSE = "false"
_NULL = "null"

ALL = (
    _ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL,
    _ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS
)

PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "|".join(ALL))


SPEC_TYPES = {
    'adhoc': {
        'action_key': 'base',
        'input_key': 'base-input'
    },
    'task': {
        'action_key': 'action',
        'input_key': 'input'
    }
}

JINJA_WITH_ST2KV = '{{.*?st2kv\..*?}}'

JINJA_WITH_ST2KV_PTRN = re.compile(JINJA_WITH_ST2KV)
JINJA_WITH_LOCAL_CTX = '{{.*?_\..*?}}'

JINJA_WITH_LOCAL_CTX_PTRN = re.compile(JINJA_WITH_LOCAL_CTX)


def _parse_cmd_and_input(cmd_str):
    cmd_matcher = CMD_PTRN.search(cmd_str)

    if not cmd_matcher:
        raise ValueError("Invalid action/workflow task property: %s" % cmd_str)

    cmd = cmd_matcher.group()

    params = {}
    for k, v in re.findall(PARAMS_PTRN, cmd_str):
        # Remove embracing quotes.
        v = v.strip()
        if v[0] == '"' or v[0] == "'":
            v = v[1:-1]
        else:
            try:
                v = json.loads(v)
            except Exception:
                pass

        params[k] = v

    return cmd, params


def _merge_dicts(left, right):
    if left is None:
        return right

    if right is None:
        return left

    for k, v in right.iteritems():
        if k not in left:
            left[k] = v
        else:
            left_v = left[k]

            if isinstance(left_v, dict) and isinstance(v, dict):
                _merge_dicts(left_v, v)

    return left


def _eval_inline_params(spec, action_key, input_key):
    action_str = spec.get(action_key)
    command, inputs = _parse_cmd_and_input(action_str)
    if inputs:
        spec[action_key] = command
        if input_key not in spec:
            spec[input_key] = {}
            _merge_dicts(spec[input_key], inputs)


def _validate_action_parameters(name, action, action_params):
    requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params)

    if requires:
        raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": '
                                          '"%s"' % (name, action.ref, '", "'.join(requires)))

    if unexpected:
        raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": '
                                          '"%s"' % (name, action.ref, '", "'.join(unexpected)))


def _transform_action_param(action_ref, param_name, param_value):
    st2kv_matches = JINJA_WITH_ST2KV_PTRN.findall(param_value)
    local_ctx_matches = JINJA_WITH_LOCAL_CTX_PTRN.findall(param_value)

    if st2kv_matches and local_ctx_matches:
        raise WorkflowDefinitionException('Parameter "%s" for action "%s" containing '
                                          'references to both local context (i.e. _.var1) '
                                          'and st2kv (i.e. st2kv.system.var1) is not '
                                          'supported.' % (param_name, action_ref))

    if st2kv_matches:
        return '{% raw %}' + param_value + '{% endraw %}'
    else:
        return param_value


def _transform_action(name, spec):

    action_key, input_key = None, None

    for spec_type, spec_meta in six.iteritems(SPEC_TYPES):
        if spec_meta['action_key'] in spec:
            action_key = spec_meta['action_key']
            input_key = spec_meta['input_key']
            break

    if not action_key:
        return

    if spec[action_key] == 'st2.callback':
        raise WorkflowDefinitionException('st2.callback is deprecated.')

    # Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2})
    # and split it to action name and input dict as illustrated below.
    #
    # action: some_action
    # input:
    #   var1: <% $.value1 %>
    #   var2: <% $.value2 %>
    #
    # This step to separate the action name and the input parameters is required
    # to wrap them with the st2.action proxy.
    #
    # action: st2.action
    # input:
    #   ref: some_action
    #   parameters:
    #     var1: <% $.value1 %>
    #     var2: <% $.value2 %>
    _eval_inline_params(spec, action_key, input_key)

    transformed = (spec[action_key] == 'st2.action')

    action_ref = spec[input_key]['ref'] if transformed else spec[action_key]

    action = None

    # Identify if action is a registered StackStorm action.
    if action_ref and ResourceReference.is_resource_reference(action_ref):
        action = action_utils.get_action_by_ref(ref=action_ref)

    # If action is a registered StackStorm action, then wrap the
    # action with the st2 proxy and validate the action input.
    if action:
        if not transformed:
            spec[action_key] = 'st2.action'
            action_input = spec.get(input_key)
            spec[input_key] = {'ref': action_ref}
            if action_input:
                spec[input_key]['parameters'] = action_input

        action_input = spec.get(input_key, {})
        action_params = action_input.get('parameters', {})
        _validate_action_parameters(name, action, action_params)

        xformed_action_params = {}

        for param_name in action_params.keys():
            param_value = copy.deepcopy(action_params[param_name])
            xformed_param_value = _transform_action_param(
                action_ref, param_name, param_value)
            xformed_action_params[param_name] = xformed_param_value

        if xformed_action_params != action_params:
            spec[input_key]['parameters'] = xformed_action_params


def transform_definition(definition):
    # If definition is a dictionary, there is no need to load from YAML.
    is_dict = isinstance(definition, dict)
    spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition)

    # Transform adhoc actions
    for action_name, action_spec in six.iteritems(spec.get('actions', {})):
        _transform_action(action_name, action_spec)

    # Determine if definition is a workbook or workflow
    is_workbook = 'workflows' in spec

    # Transform tasks
    if is_workbook:
        for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})):
            if 'tasks' in workflow_spec:
                for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')):
                    _transform_action(task_name, task_spec)
    else:
        for key, value in six.iteritems(spec):
            if 'tasks' in value:
                for task_name, task_spec in six.iteritems(value.get('tasks')):
                    _transform_action(task_name, task_spec)

    # Return the same type as original input.
    return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False)


def retry_on_exceptions(exc):
    LOG.warning('Determining if %s should be retried...', type(exc))

    is_connection_error = isinstance(exc, requests.exceptions.ConnectionError)
    is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message
    is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message
    retrying = is_connection_error or is_duplicate_error or is_messaging_error

    if retrying:
        LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc))

    return retrying


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import copy
17			import json
18			import re
19
20			import requests
21			import six
22			import yaml
23			from mistralclient.api.base import APIException
24
25			from st2common.exceptions.workflow import WorkflowDefinitionException
26			from st2common import log as logging
27			from st2common.models.system.common import ResourceReference
28			from st2common.models.utils import action_param_utils
29			from st2common.util import action_db as action_utils
30
31
32			LOG = logging.getLogger(__name__)
33
34
35			CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*")
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\w` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\.` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
36
37			INLINE_YAQL = '<%.*?%>'
38			_ALL_IN_BRACKETS = "\[.\]\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\[` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\]` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
39			_ALL_IN_QUOTES = "\"[^\"]\"\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
40			_ALL_IN_APOSTROPHES = "'[^']'\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
41			_DIGITS = "\d+"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\d` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
42			_TRUE = "true"
43			_FALSE = "false"
44			_NULL = "null"
45
46			ALL = (
47			_ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL,
48			_ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS
49			)
50
51			PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "\|".join(ALL))
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\w` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
52
53			SPEC_TYPES = {
54			'adhoc': {
55			'action_key': 'base',
56			'input_key': 'base-input'
57			},
58			'task': {
59			'action_key': 'action',
60			'input_key': 'input'
61			}
62			}
63
64			JINJA_WITH_ST2KV = '{{.?st2kv\..?}}'
			0 ignored issues – show Bug introduced 2017-01-10 01:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\.` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
65			JINJA_WITH_ST2KV_PTRN = re.compile(JINJA_WITH_ST2KV)
66			JINJA_WITH_LOCAL_CTX = '{{.?_\..?}}'
			0 ignored issues – show Bug introduced 2017-01-10 20:29 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\.` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
67			JINJA_WITH_LOCAL_CTX_PTRN = re.compile(JINJA_WITH_LOCAL_CTX)
68
69
70			def _parse_cmd_and_input(cmd_str):
71			cmd_matcher = CMD_PTRN.search(cmd_str)
72
73			if not cmd_matcher:
74			raise ValueError("Invalid action/workflow task property: %s" % cmd_str)
75
76			cmd = cmd_matcher.group()
77
78			params = {}
79			for k, v in re.findall(PARAMS_PTRN, cmd_str):
80			# Remove embracing quotes.
81			v = v.strip()
82			if v[0] == '"' or v[0] == "'":
83			v = v[1:-1]
84			else:
85			try:
86			v = json.loads(v)
87			except Exception:
88			pass
89
90			params[k] = v
91
92			return cmd, params
93
94
95			def _merge_dicts(left, right):
96			if left is None:
97			return right
98
99			if right is None:
100			return left
101
102			for k, v in right.iteritems():
103			if k not in left:
104			left[k] = v
105			else:
106			left_v = left[k]
107
108			if isinstance(left_v, dict) and isinstance(v, dict):
109			_merge_dicts(left_v, v)
110
111			return left
112
113
114			def _eval_inline_params(spec, action_key, input_key):
115			action_str = spec.get(action_key)
116			command, inputs = _parse_cmd_and_input(action_str)
117			if inputs:
118			spec[action_key] = command
119			if input_key not in spec:
120			spec[input_key] = {}
121			_merge_dicts(spec[input_key], inputs)
122
123
124			def _validate_action_parameters(name, action, action_params):
125			requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params)
126
127			if requires:
128			raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": '
129			'"%s"' % (name, action.ref, '", "'.join(requires)))
130
131			if unexpected:
132			raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": '
133			'"%s"' % (name, action.ref, '", "'.join(unexpected)))
134
135
136			def _transform_action_param(action_ref, param_name, param_value):
137			st2kv_matches = JINJA_WITH_ST2KV_PTRN.findall(param_value)
138			local_ctx_matches = JINJA_WITH_LOCAL_CTX_PTRN.findall(param_value)
139
140			if st2kv_matches and local_ctx_matches:
141			raise WorkflowDefinitionException('Parameter "%s" for action "%s" containing '
142			'references to both local context (i.e. _.var1) '
143			'and st2kv (i.e. st2kv.system.var1) is not '
144			'supported.' % (param_name, action_ref))
145
146			if st2kv_matches:
147			return '{% raw %}' + param_value + '{% endraw %}'
148			else:
149			return param_value
150
151
152			def _transform_action(name, spec):
153
154			action_key, input_key = None, None
155
156			for spec_type, spec_meta in six.iteritems(SPEC_TYPES):
157			if spec_meta['action_key'] in spec:
158			action_key = spec_meta['action_key']
159			input_key = spec_meta['input_key']
160			break
161
162			if not action_key:
163			return
164
165			if spec[action_key] == 'st2.callback':
166			raise WorkflowDefinitionException('st2.callback is deprecated.')
167
168			# Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2})
169			# and split it to action name and input dict as illustrated below.
170			#
171			# action: some_action
172			# input:
173			# var1: <% $.value1 %>
174			# var2: <% $.value2 %>
175			#
176			# This step to separate the action name and the input parameters is required
177			# to wrap them with the st2.action proxy.
178			#
179			# action: st2.action
180			# input:
181			# ref: some_action
182			# parameters:
183			# var1: <% $.value1 %>
184			# var2: <% $.value2 %>
185			_eval_inline_params(spec, action_key, input_key)
186
187			transformed = (spec[action_key] == 'st2.action')
188
189			action_ref = spec[input_key]['ref'] if transformed else spec[action_key]
190
191			action = None
192
193			# Identify if action is a registered StackStorm action.
194			if action_ref and ResourceReference.is_resource_reference(action_ref):
195			action = action_utils.get_action_by_ref(ref=action_ref)
196
197			# If action is a registered StackStorm action, then wrap the
198			# action with the st2 proxy and validate the action input.
199			if action:
200			if not transformed:
201			spec[action_key] = 'st2.action'
202			action_input = spec.get(input_key)
203			spec[input_key] = {'ref': action_ref}
204			if action_input:
205			spec[input_key]['parameters'] = action_input
206
207			action_input = spec.get(input_key, {})
208			action_params = action_input.get('parameters', {})
209			_validate_action_parameters(name, action, action_params)
210
211			xformed_action_params = {}
212
213			for param_name in action_params.keys():
214			param_value = copy.deepcopy(action_params[param_name])
215			xformed_param_value = _transform_action_param(
216			action_ref, param_name, param_value)
217			xformed_action_params[param_name] = xformed_param_value
218
219			if xformed_action_params != action_params:
220			spec[input_key]['parameters'] = xformed_action_params
221
222
223			def transform_definition(definition):
224			# If definition is a dictionary, there is no need to load from YAML.
225			is_dict = isinstance(definition, dict)
226			spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition)
227
228			# Transform adhoc actions
229			for action_name, action_spec in six.iteritems(spec.get('actions', {})):
230			_transform_action(action_name, action_spec)
231
232			# Determine if definition is a workbook or workflow
233			is_workbook = 'workflows' in spec
234
235			# Transform tasks
236			if is_workbook:
237			for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})):
238			if 'tasks' in workflow_spec:
239			for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')):
240			_transform_action(task_name, task_spec)
241			else:
242			for key, value in six.iteritems(spec):
243			if 'tasks' in value:
244			for task_name, task_spec in six.iteritems(value.get('tasks')):
245			_transform_action(task_name, task_spec)
246
247			# Return the same type as original input.
248			return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False)
249
250
251			def retry_on_exceptions(exc):
252			LOG.warning('Determining if %s should be retried...', type(exc))
253
254			is_connection_error = isinstance(exc, requests.exceptions.ConnectionError)
255			is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message
256			is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message
257			retrying = is_connection_error or is_duplicate_error or is_messaging_error
258
259			if retrying:
260			LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc))
261
262			return retrying
263

StackStorm / st2

Pull Request — master (#3154)

_transform_action() F

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like