render_live_params() - Code Metrics - Inspection of "Merge pull request #3867 from StackStorm/local_rem..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 6aeca2...dec5f2 )

unknown

created 2017-12-01 11:00 UTC

render_live_params() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	3
dl	0
loc	24
rs	8.9713
c	2
b	0
f	0

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import re
import six
import networkx as nx

from jinja2 import meta
from st2common import log as logging
from st2common.util.config_loader import get_config
from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
from st2common.constants.pack import PACK_CONFIG_CONTEXT_KV_PREFIX
from st2common.constants.keyvalue import DATASTORE_PARENT_SCOPE, SYSTEM_SCOPE, FULL_SYSTEM_SCOPE
from st2common.exceptions.param import ParamException
from st2common.services.keyvalues import KeyValueLookup
from st2common.util.casts import get_cast
from st2common.util.compat import to_unicode
from st2common.util import jinja as jinja_utils


LOG = logging.getLogger(__name__)
ENV = jinja_utils.get_jinja_environment()

__all__ = [
    'render_live_params',
    'render_final_params',
]


def _split_params(runner_parameters, action_parameters, mixed_params):
    def pf(params, skips):
        result = {k: v for k, v in six.iteritems(mixed_params)
                  if k in params and k not in skips}
        return result
    return (pf(runner_parameters, {}), pf(action_parameters, runner_parameters))


def _cast_params(rendered, parameter_schemas):
    '''
    It's just here to make tests happy
    '''
    casted_params = {}
    for k, v in six.iteritems(rendered):
        casted_params[k] = _cast(v, parameter_schemas[k] or {})
    return casted_params


def _cast(v, parameter_schema):
    if v is None or not parameter_schema:
        return v

    parameter_type = parameter_schema.get('type', None)
    if not parameter_type:
        return v

    cast = get_cast(cast_type=parameter_type)
    if not cast:
        return v

    return cast(v)


def _create_graph(action_context, config):
    '''
    Creates a generic directed graph for depencency tree and fills it with basic context variables
    '''
    G = nx.DiGraph()
    system_keyvalue_context = {SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)}
    G.add_node(DATASTORE_PARENT_SCOPE, value=system_keyvalue_context)
    G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
    G.add_node(PACK_CONFIG_CONTEXT_KV_PREFIX, value=config)
    return G


def _process(G, name, value):
    '''
    Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
    '''
    # Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
    # Instead we're just assuming every string to be a unicode string
    if isinstance(value, str):
        value = to_unicode(value)

    complex_value_str = None
    if isinstance(value, list) or isinstance(value, dict):
        complex_value_str = str(value)

    is_jinja_expr = (
        jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
            complex_value_str
        )
    )

    if is_jinja_expr:
        G.add_node(name, template=value)

        template_ast = ENV.parse(value)
        LOG.debug('Template ast: %s', template_ast)
        # Dependencies of the node represent jinja variables used in the template
        # We're connecting nodes with an edge for every depencency to traverse them
        # in the right order and also make sure that we don't have missing or cyclic
        # dependencies upfront.
        dependencies = meta.find_undeclared_variables(template_ast)
        LOG.debug('Dependencies: %s', dependencies)
        if dependencies:
            for dependency in dependencies:
                G.add_edge(dependency, name)
    else:
        G.add_node(name, value=value)


def _process_defaults(G, schemas):
    '''
    Process dependencies for parameters default values in the order schemas are defined.
    '''
    for schema in schemas:
        for name, value in six.iteritems(schema):
            absent = name not in G.node
            is_none = G.node.get(name, {}).get('value') is None
            immutable = value.get('immutable', False)
            if absent or is_none or immutable:
                _process(G, name, value.get('default'))


def _validate(G):
    '''
    Validates dependency graph to ensure it has no missing or cyclic dependencies
    '''
    for name in G.nodes():
        if 'value' not in G.node[name] and 'template' not in G.node[name]:
            msg = 'Dependecy unsatisfied in %s' % name
            raise ParamException(msg)

    if not nx.is_directed_acyclic_graph(G):
        msg = 'Cyclic dependecy found'
        raise ParamException(msg)


def _render(node, render_context):
    '''
    Render the node depending on its type
    '''
    if 'template' in node:
        complex_type = False

        if isinstance(node['template'], list) or isinstance(node['template'], dict):
            node['template'] = json.dumps(node['template'])

            # Finds occourances of "{{variable}}" and adds `to_complex` filter
            # so types are honored. If it doesn't follow that syntax then it's
            # rendered as a string.
            node['template'] = re.sub(
                r'"{{([A-z0-9_-]+)}}"', r'{{\1 | to_complex}}',
                node['template']
            )
            LOG.debug('Rendering complex type: %s', node['template'])
            complex_type = True

        LOG.debug('Rendering node: %s with context: %s', node, render_context)

        result = ENV.from_string(str(node['template'])).render(render_context)

        LOG.debug('Render complete: %s', result)

        if complex_type:
            result = json.loads(result)
            LOG.debug('Complex Type Rendered: %s', result)

        return result
    if 'value' in node:
        return node['value']


def _resolve_dependencies(G):
    '''
    Traverse the dependency graph starting from resolved nodes
    '''
    context = {}
    for name in nx.topological_sort(G):
        node = G.node[name]
        try:
            context[name] = _render(node, context)

        except Exception as e:
            LOG.debug('Failed to render %s: %s', name, e, exc_info=True)
            msg = 'Failed to render parameter "%s": %s' % (name, str(e))
            raise ParamException(msg)

    return context


def _cast_params_from(params, context, schemas):
    '''
    Pick a list of parameters from context and cast each of them according to the schemas provided
    '''
    result = {}
    for name in params:
        param_schema = {}
        for schema in schemas:
            if name in schema:
                param_schema = schema[name]
        result[name] = _cast(context[name], param_schema)
    return result


def render_live_params(runner_parameters, action_parameters, params, action_context,
                       additional_contexts=None):
    '''
    Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a
    dict of plain rendered parameters.
    '''
    additional_contexts = additional_contexts or {}
    config = get_config(action_context.get('pack'), action_context.get('user'))

    G = _create_graph(action_context, config)

    # Additional contexts are applied after all other contexts (see _create_graph), but before any
    # of the dependencies have been resolved.
    for name, value in six.iteritems(additional_contexts):
        G.add_node(name, value=value)

    [_process(G, name, value) for name, value in six.iteritems(params)]
    _process_defaults(G, [action_parameters, runner_parameters])
    _validate(G)

    context = _resolve_dependencies(G)
    live_params = _cast_params_from(params, context, [action_parameters, runner_parameters])

    return live_params


def render_final_params(runner_parameters, action_parameters, params, action_context):
    '''
    Renders missing parameters required for action to execute. Treats parameters from the dict as
    plain values instead of trying to render them again. Returns dicts for action and runner
    parameters.
    '''
    config = get_config(action_context.get('pack'), action_context.get('user'))

    G = _create_graph(action_context, config)

    # by that point, all params should already be resolved so any template should be treated value
    [G.add_node(name, value=value) for name, value in six.iteritems(params)]
    _process_defaults(G, [action_parameters, runner_parameters])
    _validate(G)

    context = _resolve_dependencies(G)
    context = _cast_params_from(context, context, [action_parameters, runner_parameters])

    return _split_params(runner_parameters, action_parameters, context)


def get_finalized_params(runnertype_parameter_info, action_parameter_info, liveaction_parameters,
                         action_context):
    '''
    Left here to keep tests running. Later we would need to split tests so they start testing each
    function separately.
    '''
    params = render_live_params(runnertype_parameter_info, action_parameter_info,
                                liveaction_parameters, action_context)
    return render_final_params(runnertype_parameter_info, action_parameter_info, params,
                               action_context)


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import json
17			import re
18			import six
19			import networkx as nx
20
21			from jinja2 import meta
22			from st2common import log as logging
23			from st2common.util.config_loader import get_config
24			from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
25			from st2common.constants.pack import PACK_CONFIG_CONTEXT_KV_PREFIX
26			from st2common.constants.keyvalue import DATASTORE_PARENT_SCOPE, SYSTEM_SCOPE, FULL_SYSTEM_SCOPE
27			from st2common.exceptions.param import ParamException
28			from st2common.services.keyvalues import KeyValueLookup
29			from st2common.util.casts import get_cast
30			from st2common.util.compat import to_unicode
31			from st2common.util import jinja as jinja_utils
32
33
34			LOG = logging.getLogger(__name__)
35			ENV = jinja_utils.get_jinja_environment()
36
37			__all__ = [
38			'render_live_params',
39			'render_final_params',
40			]
41
42
43			def _split_params(runner_parameters, action_parameters, mixed_params):
44			def pf(params, skips):
45			result = {k: v for k, v in six.iteritems(mixed_params)
46			if k in params and k not in skips}
47			return result
48			return (pf(runner_parameters, {}), pf(action_parameters, runner_parameters))
49
50
51			def _cast_params(rendered, parameter_schemas):
52			'''
53			It's just here to make tests happy
54			'''
55			casted_params = {}
56			for k, v in six.iteritems(rendered):
57			casted_params[k] = _cast(v, parameter_schemas[k] or {})
58			return casted_params
59
60
61			def _cast(v, parameter_schema):
62			if v is None or not parameter_schema:
63			return v
64
65			parameter_type = parameter_schema.get('type', None)
66			if not parameter_type:
67			return v
68
69			cast = get_cast(cast_type=parameter_type)
70			if not cast:
71			return v
72
73			return cast(v)
74
75
76			def _create_graph(action_context, config):
77			'''
78			Creates a generic directed graph for depencency tree and fills it with basic context variables
79			'''
80			G = nx.DiGraph()
81			system_keyvalue_context = {SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)}
82			G.add_node(DATASTORE_PARENT_SCOPE, value=system_keyvalue_context)
83			G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
84			G.add_node(PACK_CONFIG_CONTEXT_KV_PREFIX, value=config)
85			return G
86
87
88			def _process(G, name, value):
89			'''
90			Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
91			'''
92			# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
93			# Instead we're just assuming every string to be a unicode string
94			if isinstance(value, str):
95			value = to_unicode(value)
96
97			complex_value_str = None
98			if isinstance(value, list) or isinstance(value, dict):
99			complex_value_str = str(value)
100
101			is_jinja_expr = (
102			jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
103			complex_value_str
104			)
105			)
106
107			if is_jinja_expr:
108			G.add_node(name, template=value)
109
110			template_ast = ENV.parse(value)
111			LOG.debug('Template ast: %s', template_ast)
112			# Dependencies of the node represent jinja variables used in the template
113			# We're connecting nodes with an edge for every depencency to traverse them
114			# in the right order and also make sure that we don't have missing or cyclic
115			# dependencies upfront.
116			dependencies = meta.find_undeclared_variables(template_ast)
117			LOG.debug('Dependencies: %s', dependencies)
118			if dependencies:
119			for dependency in dependencies:
120			G.add_edge(dependency, name)
121			else:
122			G.add_node(name, value=value)
123
124
125			def _process_defaults(G, schemas):
126			'''
127			Process dependencies for parameters default values in the order schemas are defined.
128			'''
129			for schema in schemas:
130			for name, value in six.iteritems(schema):
131			absent = name not in G.node
132			is_none = G.node.get(name, {}).get('value') is None
133			immutable = value.get('immutable', False)
134			if absent or is_none or immutable:
135			_process(G, name, value.get('default'))
136
137
138			def _validate(G):
139			'''
140			Validates dependency graph to ensure it has no missing or cyclic dependencies
141			'''
142			for name in G.nodes():
143			if 'value' not in G.node[name] and 'template' not in G.node[name]:
144			msg = 'Dependecy unsatisfied in %s' % name
145			raise ParamException(msg)
146
147			if not nx.is_directed_acyclic_graph(G):
148			msg = 'Cyclic dependecy found'
149			raise ParamException(msg)
150
151
152			def _render(node, render_context):
153			'''
154			Render the node depending on its type
155			'''
156			if 'template' in node:
157			complex_type = False
158
159			if isinstance(node['template'], list) or isinstance(node['template'], dict):
160			node['template'] = json.dumps(node['template'])
161
162			# Finds occourances of "{{variable}}" and adds `to_complex` filter
163			# so types are honored. If it doesn't follow that syntax then it's
164			# rendered as a string.
165			node['template'] = re.sub(
166			r'"{{([A-z0-9_-]+)}}"', r'{{\1 \| to_complex}}',
167			node['template']
168			)
169			LOG.debug('Rendering complex type: %s', node['template'])
170			complex_type = True
171
172			LOG.debug('Rendering node: %s with context: %s', node, render_context)
173
174			result = ENV.from_string(str(node['template'])).render(render_context)
175
176			LOG.debug('Render complete: %s', result)
177
178			if complex_type:
179			result = json.loads(result)
180			LOG.debug('Complex Type Rendered: %s', result)
181
182			return result
183			if 'value' in node:
184			return node['value']
185
186
187			def _resolve_dependencies(G):
188			'''
189			Traverse the dependency graph starting from resolved nodes
190			'''
191			context = {}
192			for name in nx.topological_sort(G):
193			node = G.node[name]
194			try:
195			context[name] = _render(node, context)
196
197			except Exception as e:
198			LOG.debug('Failed to render %s: %s', name, e, exc_info=True)
199			msg = 'Failed to render parameter "%s": %s' % (name, str(e))
200			raise ParamException(msg)
201
202			return context
203
204
205			def _cast_params_from(params, context, schemas):
206			'''
207			Pick a list of parameters from context and cast each of them according to the schemas provided
208			'''
209			result = {}
210			for name in params:
211			param_schema = {}
212			for schema in schemas:
213			if name in schema:
214			param_schema = schema[name]
215			result[name] = _cast(context[name], param_schema)
216			return result
217
218
219			def render_live_params(runner_parameters, action_parameters, params, action_context,
220			additional_contexts=None):
221			'''
222			Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a
223			dict of plain rendered parameters.
224			'''
225			additional_contexts = additional_contexts or {}
226			config = get_config(action_context.get('pack'), action_context.get('user'))
227
228			G = _create_graph(action_context, config)
229
230			# Additional contexts are applied after all other contexts (see _create_graph), but before any
231			# of the dependencies have been resolved.
232			for name, value in six.iteritems(additional_contexts):
233			G.add_node(name, value=value)
234
235			[_process(G, name, value) for name, value in six.iteritems(params)]
236			_process_defaults(G, [action_parameters, runner_parameters])
237			_validate(G)
238
239			context = _resolve_dependencies(G)
240			live_params = _cast_params_from(params, context, [action_parameters, runner_parameters])
241
242			return live_params
243
244
245			def render_final_params(runner_parameters, action_parameters, params, action_context):
246			'''
247			Renders missing parameters required for action to execute. Treats parameters from the dict as
248			plain values instead of trying to render them again. Returns dicts for action and runner
249			parameters.
250			'''
251			config = get_config(action_context.get('pack'), action_context.get('user'))
252
253			G = _create_graph(action_context, config)
254
255			# by that point, all params should already be resolved so any template should be treated value
256			[G.add_node(name, value=value) for name, value in six.iteritems(params)]
257			_process_defaults(G, [action_parameters, runner_parameters])
258			_validate(G)
259
260			context = _resolve_dependencies(G)
261			context = _cast_params_from(context, context, [action_parameters, runner_parameters])
262
263			return _split_params(runner_parameters, action_parameters, context)
264
265
266			def get_finalized_params(runnertype_parameter_info, action_parameter_info, liveaction_parameters,
267			action_context):
268			'''
269			Left here to keep tests running. Later we would need to split tests so they start testing each
270			function separately.
271			'''
272			params = render_live_params(runnertype_parameter_info, action_parameter_info,
273			liveaction_parameters, action_context)
274			return render_final_params(runnertype_parameter_info, action_parameter_info, params,
275			action_context)
276

StackStorm / st2

Push — master ( 6aeca2...dec5f2 )

render_live_params() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like