_render() - Code Metrics - Inspection of "Merge pull request #3018 from StackStorm/fix/compl..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( c851a6...0bb377 )

by Plexxi

created 2017-01-04 11:01 UTC

_render() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	6
c	2
b	0
f	0
dl	0
loc	33
rs	7.5384

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import re
import six
import networkx as nx

from jinja2 import meta
from st2common import log as logging
from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
from st2common.constants.keyvalue import DATASTORE_PARENT_SCOPE, SYSTEM_SCOPE
from st2common.exceptions.param import ParamException
from st2common.services.keyvalues import KeyValueLookup
from st2common.util.casts import get_cast
from st2common.util.compat import to_unicode
from st2common.util import jinja as jinja_utils


LOG = logging.getLogger(__name__)
ENV = jinja_utils.get_jinja_environment()

__all__ = [
    'render_live_params',
    'render_final_params',
]


def _split_params(runner_parameters, action_parameters, mixed_params):
    def pf(params, skips):
        result = {k: v for k, v in six.iteritems(mixed_params)
                  if k in params and k not in skips}
        return result
    return (pf(runner_parameters, {}), pf(action_parameters, runner_parameters))


def _cast_params(rendered, parameter_schemas):
    '''
    It's just here to make tests happy
    '''
    casted_params = {}
    for k, v in six.iteritems(rendered):
        casted_params[k] = _cast(v, parameter_schemas[k] or {})
    return casted_params


def _cast(v, parameter_schema):
    if v is None or not parameter_schema:
        return v

    parameter_type = parameter_schema.get('type', None)
    if not parameter_type:
        return v

    cast = get_cast(cast_type=parameter_type)
    if not cast:
        return v

    return cast(v)


def _create_graph(action_context):
    '''
    Creates a generic directed graph for depencency tree and fills it with basic context variables
    '''
    G = nx.DiGraph()
    G.add_node(SYSTEM_SCOPE, value=KeyValueLookup(scope=SYSTEM_SCOPE))
    system_keyvalue_context = {SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}
    G.add_node(DATASTORE_PARENT_SCOPE, value=system_keyvalue_context)
    G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
    return G


def _process(G, name, value):
    '''
    Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
    '''
    # Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
    # Instead we're just assuming every string to be a unicode string
    if isinstance(value, str):
        value = to_unicode(value)

    complex_value_str = None
    if isinstance(value, list) or isinstance(value, dict):
        complex_value_str = str(value)

    is_jinja_expr = (
        jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
            complex_value_str
        )
    )

    if is_jinja_expr:
        G.add_node(name, template=value)

        template_ast = ENV.parse(value)
        LOG.debug('Template ast: %s', template_ast)
        # Dependencies of the node represent jinja variables used in the template
        # We're connecting nodes with an edge for every depencency to traverse them
        # in the right order and also make sure that we don't have missing or cyclic
        # dependencies upfront.
        dependencies = meta.find_undeclared_variables(template_ast)
        LOG.debug('Dependencies: %s', dependencies)
        if dependencies:
            for dependency in dependencies:
                G.add_edge(dependency, name)
    else:
        G.add_node(name, value=value)


def _process_defaults(G, schemas):
    '''
    Process dependencies for parameters default values in the order schemas are defined.
    '''
    for schema in schemas:
        for name, value in six.iteritems(schema):
            absent = name not in G.node
            is_none = G.node.get(name, {}).get('value') is None
            immutable = value.get('immutable', False)
            if absent or is_none or immutable:
                _process(G, name, value.get('default'))


def _validate(G):
    '''
    Validates dependency graph to ensure it has no missing or cyclic dependencies
    '''
    for name in G.nodes():
        if 'value' not in G.node[name] and 'template' not in G.node[name]:
            msg = 'Dependecy unsatisfied in %s' % name
            raise ParamException(msg)

    if not nx.is_directed_acyclic_graph(G):
        msg = 'Cyclic dependecy found'
        raise ParamException(msg)


def _render(node, render_context):
    '''
    Render the node depending on its type
    '''
    if 'template' in node:
        complex_type = False

        if isinstance(node['template'], list) or isinstance(node['template'], dict):
            node['template'] = json.dumps(node['template'])

            # Finds occourances of "{{variable}}" and adds `to_complex` filter
            # so types are honored. If it doesn't follow that syntax then it's
            # rendered as a string.
            node['template'] = re.sub(
                r'"{{([A-z0-9_-]+)}}"', r'{{\1 | to_complex}}',
                node['template']
            )
            LOG.debug('Rendering complex type: %s', node['template'])
            complex_type = True

        LOG.debug('Rendering node: %s with context: %s', node, render_context)

        result = ENV.from_string(str(node['template'])).render(render_context)

        LOG.debug('Render complete: %s', result)

        if complex_type:
            result = json.loads(result)
            LOG.debug('Complex Type Rendered: %s', result)

        return result
    if 'value' in node:
        return node['value']


def _resolve_dependencies(G):
    '''
    Traverse the dependency graph starting from resolved nodes
    '''
    context = {}
    for name in nx.topological_sort(G):
        node = G.node[name]
        try:
            context[name] = _render(node, context)

        except Exception as e:
            LOG.debug('Failed to render %s: %s', name, e, exc_info=True)
            msg = 'Failed to render parameter "%s": %s' % (name, str(e))
            raise ParamException(msg)

    return context


def _cast_params_from(params, context, schemas):
    '''
    Pick a list of parameters from context and cast each of them according to the schemas provided
    '''
    result = {}
    for name in params:
        param_schema = {}
        for schema in schemas:
            if name in schema:
                param_schema = schema[name]
        result[name] = _cast(context[name], param_schema)
    return result


def render_live_params(runner_parameters, action_parameters, params, action_context):
    '''
    Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a
    dict of plain rendered parameters.
    '''
    G = _create_graph(action_context)

    [_process(G, name, value) for name, value in six.iteritems(params)]
    _process_defaults(G, [action_parameters, runner_parameters])
    _validate(G)

    context = _resolve_dependencies(G)
    live_params = _cast_params_from(params, context, [action_parameters, runner_parameters])

    return live_params


def render_final_params(runner_parameters, action_parameters, params, action_context):
    '''
    Renders missing parameters required for action to execute. Treats parameters from the dict as
    plain values instead of trying to render them again. Returns dicts for action and runner
    parameters.
    '''
    G = _create_graph(action_context)

    # by that point, all params should already be resolved so any template should be treated value
    [G.add_node(name, value=value) for name, value in six.iteritems(params)]
    _process_defaults(G, [action_parameters, runner_parameters])
    _validate(G)

    context = _resolve_dependencies(G)
    context = _cast_params_from(context, context, [action_parameters, runner_parameters])

    return _split_params(runner_parameters, action_parameters, context)


def get_finalized_params(runnertype_parameter_info, action_parameter_info, liveaction_parameters,
                         action_context):
    '''
    Left here to keep tests running. Later we would need to split tests so they start testing each
    function separately.
    '''
    params = render_live_params(runnertype_parameter_info, action_parameter_info,
                                liveaction_parameters, action_context)
    return render_final_params(runnertype_parameter_info, action_parameter_info, params,
                               action_context)


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import json
17			import re
18			import six
19			import networkx as nx
20
21			from jinja2 import meta
22			from st2common import log as logging
23			from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
24			from st2common.constants.keyvalue import DATASTORE_PARENT_SCOPE, SYSTEM_SCOPE
25			from st2common.exceptions.param import ParamException
26			from st2common.services.keyvalues import KeyValueLookup
27			from st2common.util.casts import get_cast
28			from st2common.util.compat import to_unicode
29			from st2common.util import jinja as jinja_utils
30
31
32			LOG = logging.getLogger(__name__)
33			ENV = jinja_utils.get_jinja_environment()
34
35			__all__ = [
36			'render_live_params',
37			'render_final_params',
38			]
39
40
41			def _split_params(runner_parameters, action_parameters, mixed_params):
42			def pf(params, skips):
43			result = {k: v for k, v in six.iteritems(mixed_params)
44			if k in params and k not in skips}
45			return result
46			return (pf(runner_parameters, {}), pf(action_parameters, runner_parameters))
47
48
49			def _cast_params(rendered, parameter_schemas):
50			'''
51			It's just here to make tests happy
52			'''
53			casted_params = {}
54			for k, v in six.iteritems(rendered):
55			casted_params[k] = _cast(v, parameter_schemas[k] or {})
56			return casted_params
57
58
59			def _cast(v, parameter_schema):
60			if v is None or not parameter_schema:
61			return v
62
63			parameter_type = parameter_schema.get('type', None)
64			if not parameter_type:
65			return v
66
67			cast = get_cast(cast_type=parameter_type)
68			if not cast:
69			return v
70
71			return cast(v)
72
73
74			def _create_graph(action_context):
75			'''
76			Creates a generic directed graph for depencency tree and fills it with basic context variables
77			'''
78			G = nx.DiGraph()
79			G.add_node(SYSTEM_SCOPE, value=KeyValueLookup(scope=SYSTEM_SCOPE))
80			system_keyvalue_context = {SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}
81			G.add_node(DATASTORE_PARENT_SCOPE, value=system_keyvalue_context)
82			G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
83			return G
84
85
86			def _process(G, name, value):
87			'''
88			Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
89			'''
90			# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
91			# Instead we're just assuming every string to be a unicode string
92			if isinstance(value, str):
93			value = to_unicode(value)
94
95			complex_value_str = None
96			if isinstance(value, list) or isinstance(value, dict):
97			complex_value_str = str(value)
98
99			is_jinja_expr = (
100			jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
101			complex_value_str
102			)
103			)
104
105			if is_jinja_expr:
106			G.add_node(name, template=value)
107
108			template_ast = ENV.parse(value)
109			LOG.debug('Template ast: %s', template_ast)
110			# Dependencies of the node represent jinja variables used in the template
111			# We're connecting nodes with an edge for every depencency to traverse them
112			# in the right order and also make sure that we don't have missing or cyclic
113			# dependencies upfront.
114			dependencies = meta.find_undeclared_variables(template_ast)
115			LOG.debug('Dependencies: %s', dependencies)
116			if dependencies:
117			for dependency in dependencies:
118			G.add_edge(dependency, name)
119			else:
120			G.add_node(name, value=value)
121
122
123			def _process_defaults(G, schemas):
124			'''
125			Process dependencies for parameters default values in the order schemas are defined.
126			'''
127			for schema in schemas:
128			for name, value in six.iteritems(schema):
129			absent = name not in G.node
130			is_none = G.node.get(name, {}).get('value') is None
131			immutable = value.get('immutable', False)
132			if absent or is_none or immutable:
133			_process(G, name, value.get('default'))
134
135
136			def _validate(G):
137			'''
138			Validates dependency graph to ensure it has no missing or cyclic dependencies
139			'''
140			for name in G.nodes():
141			if 'value' not in G.node[name] and 'template' not in G.node[name]:
142			msg = 'Dependecy unsatisfied in %s' % name
143			raise ParamException(msg)
144
145			if not nx.is_directed_acyclic_graph(G):
146			msg = 'Cyclic dependecy found'
147			raise ParamException(msg)
148
149
150			def _render(node, render_context):
151			'''
152			Render the node depending on its type
153			'''
154			if 'template' in node:
155			complex_type = False
156
157			if isinstance(node['template'], list) or isinstance(node['template'], dict):
158			node['template'] = json.dumps(node['template'])
159
160			# Finds occourances of "{{variable}}" and adds `to_complex` filter
161			# so types are honored. If it doesn't follow that syntax then it's
162			# rendered as a string.
163			node['template'] = re.sub(
164			r'"{{([A-z0-9_-]+)}}"', r'{{\1 \| to_complex}}',
165			node['template']
166			)
167			LOG.debug('Rendering complex type: %s', node['template'])
168			complex_type = True
169
170			LOG.debug('Rendering node: %s with context: %s', node, render_context)
171
172			result = ENV.from_string(str(node['template'])).render(render_context)
173
174			LOG.debug('Render complete: %s', result)
175
176			if complex_type:
177			result = json.loads(result)
178			LOG.debug('Complex Type Rendered: %s', result)
179
180			return result
181			if 'value' in node:
182			return node['value']
183
184
185			def _resolve_dependencies(G):
186			'''
187			Traverse the dependency graph starting from resolved nodes
188			'''
189			context = {}
190			for name in nx.topological_sort(G):
191			node = G.node[name]
192			try:
193			context[name] = _render(node, context)
194
195			except Exception as e:
196			LOG.debug('Failed to render %s: %s', name, e, exc_info=True)
197			msg = 'Failed to render parameter "%s": %s' % (name, str(e))
198			raise ParamException(msg)
199
200			return context
201
202
203			def _cast_params_from(params, context, schemas):
204			'''
205			Pick a list of parameters from context and cast each of them according to the schemas provided
206			'''
207			result = {}
208			for name in params:
209			param_schema = {}
210			for schema in schemas:
211			if name in schema:
212			param_schema = schema[name]
213			result[name] = _cast(context[name], param_schema)
214			return result
215
216
217			def render_live_params(runner_parameters, action_parameters, params, action_context):
218			'''
219			Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a
220			dict of plain rendered parameters.
221			'''
222			G = _create_graph(action_context)
223
224			[_process(G, name, value) for name, value in six.iteritems(params)]
225			_process_defaults(G, [action_parameters, runner_parameters])
226			_validate(G)
227
228			context = _resolve_dependencies(G)
229			live_params = _cast_params_from(params, context, [action_parameters, runner_parameters])
230
231			return live_params
232
233
234			def render_final_params(runner_parameters, action_parameters, params, action_context):
235			'''
236			Renders missing parameters required for action to execute. Treats parameters from the dict as
237			plain values instead of trying to render them again. Returns dicts for action and runner
238			parameters.
239			'''
240			G = _create_graph(action_context)
241
242			# by that point, all params should already be resolved so any template should be treated value
243			[G.add_node(name, value=value) for name, value in six.iteritems(params)]
244			_process_defaults(G, [action_parameters, runner_parameters])
245			_validate(G)
246
247			context = _resolve_dependencies(G)
248			context = _cast_params_from(context, context, [action_parameters, runner_parameters])
249
250			return _split_params(runner_parameters, action_parameters, context)
251
252
253			def get_finalized_params(runnertype_parameter_info, action_parameter_info, liveaction_parameters,
254			action_context):
255			'''
256			Left here to keep tests running. Later we would need to split tests so they start testing each
257			function separately.
258			'''
259			params = render_live_params(runnertype_parameter_info, action_parameter_info,
260			liveaction_parameters, action_context)
261			return render_final_params(runnertype_parameter_info, action_parameter_info, params,
262			action_context)
263

StackStorm / st2

Push — master ( c851a6...0bb377 )

_render() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like