st2common.util.workflow._transform_action() - Code Metrics - Inspection of "Merge pull request #2246 from StackStorm/chatops/c..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( d1f0a7...3b2ece )

by Edward

created 2015-11-30 10:28 UTC

st2common.util.workflow._transform_action() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
dl	0
loc	58
rs	4.2857
cc	11

How to fix Long Method Complexity

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import json
import re

import requests
import six
import yaml
from mistralclient.api.base import APIException

from st2common.exceptions.workflow import WorkflowDefinitionException
from st2common import log as logging
from st2common.models.system.common import ResourceReference
from st2common.models.utils import action_param_utils
from st2common.util import action_db as action_utils


LOG = logging.getLogger(__name__)


CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*")


INLINE_YAQL = '<%.*?%>'
_ALL_IN_BRACKETS = "\[.*\]\s*"

_ALL_IN_QUOTES = "\"[^\"]*\"\s*"

_ALL_IN_APOSTROPHES = "'[^']*'\s*"

_DIGITS = "\d+"

_TRUE = "true"
_FALSE = "false"
_NULL = "null"

ALL = (
    _ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL,
    _ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS
)

PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "|".join(ALL))


SPEC_TYPES = {
    'adhoc': {
        'action_key': 'base',
        'input_key': 'base-input'
    },
    'task': {
        'action_key': 'action',
        'input_key': 'input'
    }
}


def _parse_cmd_and_input(cmd_str):
    cmd_matcher = CMD_PTRN.search(cmd_str)

    if not cmd_matcher:
        raise ValueError("Invalid action/workflow task property: %s" % cmd_str)

    cmd = cmd_matcher.group()

    params = {}
    for k, v in re.findall(PARAMS_PTRN, cmd_str):
        # Remove embracing quotes.
        v = v.strip()
        if v[0] == '"' or v[0] == "'":
            v = v[1:-1]
        else:
            try:
                v = json.loads(v)
            except Exception:
                pass

        params[k] = v

    return cmd, params


def _merge_dicts(left, right):
    if left is None:
        return right

    if right is None:
        return left

    for k, v in right.iteritems():
        if k not in left:
            left[k] = v
        else:
            left_v = left[k]

            if isinstance(left_v, dict) and isinstance(v, dict):
                _merge_dicts(left_v, v)

    return left


def _eval_inline_params(spec, action_key, input_key):
    action_str = spec.get(action_key)
    command, inputs = _parse_cmd_and_input(action_str)
    if inputs:
        spec[action_key] = command
        if input_key not in spec:
            spec[input_key] = {}
            _merge_dicts(spec[input_key], inputs)


def _validate_action_parameters(name, action, action_params):
    requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params)

    if requires:
        raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": '
                                          '"%s"' % (name, action.ref, '", "'.join(requires)))

    if unexpected:
        raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": '
                                          '"%s"' % (name, action.ref, '", "'.join(unexpected)))


def _transform_action(name, spec):

    action_key, input_key = None, None

    for spec_type, spec_meta in six.iteritems(SPEC_TYPES):
        if spec_meta['action_key'] in spec:
            action_key = spec_meta['action_key']
            input_key = spec_meta['input_key']
            break

    if not action_key:
        return

    if spec[action_key] == 'st2.callback':
        raise WorkflowDefinitionException('st2.callback is deprecated.')

    # Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2})
    # and split it to action name and input dict as illustrated below.
    #
    # action: some_action
    # input:
    #   var1: <% $.value1 %>
    #   var2: <% $.value2 %>
    #
    # This step to separate the action name and the input parameters is required
    # to wrap them with the st2.action proxy.
    #
    # action: st2.action
    # input:
    #   ref: some_action
    #   parameters:
    #     var1: <% $.value1 %>
    #     var2: <% $.value2 %>
    _eval_inline_params(spec, action_key, input_key)

    transformed = (spec[action_key] == 'st2.action')

    action_ref = spec[input_key]['ref'] if transformed else spec[action_key]

    action = None

    # Identify if action is a registered StackStorm action.
    if action_ref and ResourceReference.is_resource_reference(action_ref):
        action = action_utils.get_action_by_ref(ref=action_ref)

    # If action is a registered StackStorm action, then wrap the
    # action with the st2 proxy and validate the action input.
    if action:
        if not transformed:
            spec[action_key] = 'st2.action'
            action_input = spec.get(input_key)
            spec[input_key] = {'ref': action_ref}
            if action_input:
                spec[input_key]['parameters'] = action_input

        action_input = spec.get(input_key, {})
        action_params = action_input.get('parameters', {})
        _validate_action_parameters(name, action, action_params)


def transform_definition(definition):
    # If definition is a dictionary, there is no need to load from YAML.
    is_dict = isinstance(definition, dict)
    spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition)

    # Transform adhoc actions
    for action_name, action_spec in six.iteritems(spec.get('actions', {})):
        _transform_action(action_name, action_spec)

    # Determine if definition is a workbook or workflow
    is_workbook = 'workflows' in spec

    # Transform tasks
    if is_workbook:
        for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})):
            if 'tasks' in workflow_spec:
                for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')):
                    _transform_action(task_name, task_spec)
    else:
        for key, value in six.iteritems(spec):
            if 'tasks' in value:
                for task_name, task_spec in six.iteritems(value.get('tasks')):
                    _transform_action(task_name, task_spec)

    # Return the same type as original input.
    return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False)


def retry_on_exceptions(exc):
    LOG.warning('Determining if %s should be retried...', type(exc))

    is_connection_error = isinstance(exc, requests.exceptions.ConnectionError)
    is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message
    is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message
    retrying = is_connection_error or is_duplicate_error or is_messaging_error

    if retrying:
        LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc))

    return retrying


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import copy
17			import json
18			import re
19
20			import requests
21			import six
22			import yaml
23			from mistralclient.api.base import APIException
24
25			from st2common.exceptions.workflow import WorkflowDefinitionException
26			from st2common import log as logging
27			from st2common.models.system.common import ResourceReference
28			from st2common.models.utils import action_param_utils
29			from st2common.util import action_db as action_utils
30
31
32			LOG = logging.getLogger(__name__)
33
34
35			CMD_PTRN = re.compile("^[\w\.]+[^=\s\"]*")
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\w` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\.` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
36
37			INLINE_YAQL = '<%.*?%>'
38			_ALL_IN_BRACKETS = "\[.\]\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\[` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\]` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history... Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
39			_ALL_IN_QUOTES = "\"[^\"]\"\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
40			_ALL_IN_APOSTROPHES = "'[^']'\s"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\s` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
41			_DIGITS = "\d+"
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\d` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
42			_TRUE = "true"
43			_FALSE = "false"
44			_NULL = "null"
45
46			ALL = (
47			_ALL_IN_QUOTES, _ALL_IN_APOSTROPHES, INLINE_YAQL,
48			_ALL_IN_BRACKETS, _TRUE, _FALSE, _NULL, _DIGITS
49			)
50
51			PARAMS_PTRN = re.compile("([\w]+)=(%s)" % "\|".join(ALL))
			0 ignored issues – show Bug introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report A suspicious escape sequence `\w` was found. Did you maybe forget to add an `r` prefix? Escape sequences in Python are generally interpreted according to rules similar to standard C. Only if strings are prefixed with `r` or `R` are they interpreted as regular expressions. The escape sequence that was used indicates that you might have intended to write a regular expression. Learn more about the available escape sequences. in the Python documentation. Loading history...
52
53			SPEC_TYPES = {
54			'adhoc': {
55			'action_key': 'base',
56			'input_key': 'base-input'
57			},
58			'task': {
59			'action_key': 'action',
60			'input_key': 'input'
61			}
62			}
63
64
65			def _parse_cmd_and_input(cmd_str):
66			cmd_matcher = CMD_PTRN.search(cmd_str)
67
68			if not cmd_matcher:
69			raise ValueError("Invalid action/workflow task property: %s" % cmd_str)
70
71			cmd = cmd_matcher.group()
72
73			params = {}
74			for k, v in re.findall(PARAMS_PTRN, cmd_str):
75			# Remove embracing quotes.
76			v = v.strip()
77			if v[0] == '"' or v[0] == "'":
78			v = v[1:-1]
79			else:
80			try:
81			v = json.loads(v)
82			except Exception:
83			pass
84
85			params[k] = v
86
87			return cmd, params
88
89
90			def _merge_dicts(left, right):
91			if left is None:
92			return right
93
94			if right is None:
95			return left
96
97			for k, v in right.iteritems():
98			if k not in left:
99			left[k] = v
100			else:
101			left_v = left[k]
102
103			if isinstance(left_v, dict) and isinstance(v, dict):
104			_merge_dicts(left_v, v)
105
106			return left
107
108
109			def _eval_inline_params(spec, action_key, input_key):
110			action_str = spec.get(action_key)
111			command, inputs = _parse_cmd_and_input(action_str)
112			if inputs:
113			spec[action_key] = command
114			if input_key not in spec:
115			spec[input_key] = {}
116			_merge_dicts(spec[input_key], inputs)
117
118
119			def _validate_action_parameters(name, action, action_params):
120			requires, unexpected = action_param_utils.validate_action_parameters(action.ref, action_params)
121
122			if requires:
123			raise WorkflowDefinitionException('Missing required parameters in "%s" for action "%s": '
124			'"%s"' % (name, action.ref, '", "'.join(requires)))
125
126			if unexpected:
127			raise WorkflowDefinitionException('Unexpected parameters in "%s" for action "%s": '
128			'"%s"' % (name, action.ref, '", "'.join(unexpected)))
129
130
131			def _transform_action(name, spec):
132
133			action_key, input_key = None, None
134
135			for spec_type, spec_meta in six.iteritems(SPEC_TYPES):
136			if spec_meta['action_key'] in spec:
137			action_key = spec_meta['action_key']
138			input_key = spec_meta['input_key']
139			break
140
141			if not action_key:
142			return
143
144			if spec[action_key] == 'st2.callback':
145			raise WorkflowDefinitionException('st2.callback is deprecated.')
146
147			# Convert parameters that are inline (i.e. action: some_action var1={$.value1} var2={$.value2})
148			# and split it to action name and input dict as illustrated below.
149			#
150			# action: some_action
151			# input:
152			# var1: <% $.value1 %>
153			# var2: <% $.value2 %>
154			#
155			# This step to separate the action name and the input parameters is required
156			# to wrap them with the st2.action proxy.
157			#
158			# action: st2.action
159			# input:
160			# ref: some_action
161			# parameters:
162			# var1: <% $.value1 %>
163			# var2: <% $.value2 %>
164			_eval_inline_params(spec, action_key, input_key)
165
166			transformed = (spec[action_key] == 'st2.action')
167
168			action_ref = spec[input_key]['ref'] if transformed else spec[action_key]
169
170			action = None
171
172			# Identify if action is a registered StackStorm action.
173			if action_ref and ResourceReference.is_resource_reference(action_ref):
174			action = action_utils.get_action_by_ref(ref=action_ref)
175
176			# If action is a registered StackStorm action, then wrap the
177			# action with the st2 proxy and validate the action input.
178			if action:
179			if not transformed:
180			spec[action_key] = 'st2.action'
181			action_input = spec.get(input_key)
182			spec[input_key] = {'ref': action_ref}
183			if action_input:
184			spec[input_key]['parameters'] = action_input
185
186			action_input = spec.get(input_key, {})
187			action_params = action_input.get('parameters', {})
188			_validate_action_parameters(name, action, action_params)
189
190
191			def transform_definition(definition):
192			# If definition is a dictionary, there is no need to load from YAML.
193			is_dict = isinstance(definition, dict)
194			spec = copy.deepcopy(definition) if is_dict else yaml.safe_load(definition)
195
196			# Transform adhoc actions
197			for action_name, action_spec in six.iteritems(spec.get('actions', {})):
198			_transform_action(action_name, action_spec)
199
200			# Determine if definition is a workbook or workflow
201			is_workbook = 'workflows' in spec
202
203			# Transform tasks
204			if is_workbook:
205			for workflow_name, workflow_spec in six.iteritems(spec.get('workflows', {})):
206			if 'tasks' in workflow_spec:
207			for task_name, task_spec in six.iteritems(workflow_spec.get('tasks')):
208			_transform_action(task_name, task_spec)
209			else:
210			for key, value in six.iteritems(spec):
211			if 'tasks' in value:
212			for task_name, task_spec in six.iteritems(value.get('tasks')):
213			_transform_action(task_name, task_spec)
214
215			# Return the same type as original input.
216			return spec if is_dict else yaml.safe_dump(spec, default_flow_style=False)
217
218
219			def retry_on_exceptions(exc):
220			LOG.warning('Determining if %s should be retried...', type(exc))
221
222			is_connection_error = isinstance(exc, requests.exceptions.ConnectionError)
223			is_duplicate_error = isinstance(exc, APIException) and 'Duplicate' in exc.error_message
224			is_messaging_error = isinstance(exc, APIException) and 'MessagingTimeout' in exc.error_message
225			retrying = is_connection_error or is_duplicate_error or is_messaging_error
226
227			if retrying:
228			LOG.warning('Retrying Mistral API invocation on exception type %s.', type(exc))
229
230			return retrying
231

StackStorm / st2

Push — master ( d1f0a7...3b2ece )

st2common.util.workflow._transform_action() D

Complexity

Size

Duplication

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like