_resolve_dependencies() - Code Metrics - Inspection of "Merge pull request #2906 from StackStorm/yaml_defi..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — master ( 07d26d...59f2fc )

by Plexxi

created 2016-09-16 10:42 UTC

_resolve_dependencies() F

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	9
c	1
b	0
f	0
dl	0
loc	36
rs	3

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import six
import networkx as nx

from jinja2 import meta
from st2common import log as logging
from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
from st2common.constants.keyvalue import SYSTEM_SCOPE
from st2common.exceptions.param import ParamException
from st2common.services.keyvalues import KeyValueLookup
from st2common.util.casts import get_cast
from st2common.util.compat import to_unicode
from st2common.util import jinja as jinja_utils


LOG = logging.getLogger(__name__)
ENV = jinja_utils.get_jinja_environment()

__all__ = [
    'render_live_params',
    'render_final_params',
]


def _split_params(runner_parameters, action_parameters, mixed_params):
    def pf(params, skips):
        result = {k: v for k, v in six.iteritems(mixed_params)
                  if k in params and k not in skips}
        return result
    return (pf(runner_parameters, {}), pf(action_parameters, runner_parameters))


def _cast_params(rendered, parameter_schemas):
    '''
    It's just here to make tests happy
    '''
    casted_params = {}
    for k, v in six.iteritems(rendered):
        casted_params[k] = _cast(v, parameter_schemas[k] or {})
    return casted_params


def _cast(v, parameter_schema):
    if v is None or not parameter_schema:
        return v

    parameter_type = parameter_schema.get('type', None)
    if not parameter_type:
        return v

    cast = get_cast(cast_type=parameter_type)
    if not cast:
        return v

    return cast(v)


def _create_graph(action_context):
    '''
    Creates a generic directed graph for depencency tree and fills it with basic context variables
    '''
    G = nx.DiGraph()
    G.add_node(SYSTEM_SCOPE, value=KeyValueLookup(scope=SYSTEM_SCOPE))
    G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
    return G


def _process(G, name, value):
    '''
    Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
    '''
    # Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
    # Instead we're just assuming every string to be a unicode string
    if isinstance(value, str):
        value = to_unicode(value)

    complex_value_str = None
    if isinstance(value, list) or isinstance(value, dict):
        complex_value_str = str(value)

    is_jinja_expr = (
        jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
            complex_value_str
        )
    )

    if is_jinja_expr:
        G.add_node(name, template=value)

        template_ast = ENV.parse(value)
        LOG.debug('Template ast: %s', template_ast)
        # Dependencies of the node represent jinja variables used in the template
        # We're connecting nodes with an edge for every depencency to traverse them
        # in the right order and also make sure that we don't have missing or cyclic
        # dependencies upfront.
        dependencies = meta.find_undeclared_variables(template_ast)
        LOG.debug('Dependencies: %s', dependencies)
        if dependencies:
            for dependency in dependencies:
                G.add_edge(dependency, name)
    else:
        G.add_node(name, value=value)


def _process_defaults(G, schemas):
    '''
    Process dependencies for parameters default values in the order schemas are defined.
    '''
    for schema in schemas:
        for name, value in six.iteritems(schema):
            absent = name not in G.node
            is_none = G.node.get(name, {}).get('value') is None
            immutable = value.get('immutable', False)
            if absent or is_none or immutable:
                _process(G, name, value.get('default'))


def _validate(G):
    '''
    Validates dependency graph to ensure it has no missing or cyclic dependencies
    '''
    for name in G.nodes():
        if 'value' not in G.node[name] and 'template' not in G.node[name]:
            msg = 'Dependecy unsatisfied in %s' % name
            raise ParamException(msg)

    if not nx.is_directed_acyclic_graph(G):
        msg = 'Cyclic dependecy found'
        raise ParamException(msg)


def _render(node, render_context):
    '''
    Render the node depending on its type
    '''
    if 'template' in node:
        LOG.debug('Rendering node: %s with context: %s', node, render_context)
        return ENV.from_string(node['template']).render(render_context)
    if 'value' in node:
        return node['value']


def _resolve_dependencies(G):
    '''
    Traverse the dependency graph starting from resolved nodes
    '''
    context = {}
    for name in nx.topological_sort(G):
        node = G.node[name]
        try:
            template = node.get('template', None)

            # Special case for non simple types which contains Jinja notation (lists, dicts)
            if 'template' in node and isinstance(template, (list, dict)):
                if isinstance(template, list):
                    rendered_list = list()

                    for template in G.node[name]['template']:
                        rendered_list.append(
                            _render(dict(template=template), context)
                        )
                    context[name] = rendered_list
                elif isinstance(template, dict):
                    rendered_dict = dict()

                    for key, value in G.node[name]['template'].items():
                        value = _render(dict(template=value), context)
                        rendered_dict[key] = value

                    context[name] = rendered_dict
            else:
                context[name] = _render(node, context)
        except Exception as e:
            LOG.debug('Failed to render %s: %s', name, e, exc_info=True)
            msg = 'Failed to render parameter "%s": %s' % (name, str(e))
            raise ParamException(msg)

    return context


def _cast_params_from(params, context, schemas):
    '''
    Pick a list of parameters from context and cast each of them according to the schemas provided
    '''
    result = {}
    for name in params:
        param_schema = {}
        for schema in schemas:
            if name in schema:
                param_schema = schema[name]
        result[name] = _cast(context[name], param_schema)
    return result


def render_live_params(runner_parameters, action_parameters, params, action_context):
    '''
    Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a
    dict of plain rendered parameters.
    '''
    G = _create_graph(action_context)

    [_process(G, name, value) for name, value in six.iteritems(params)]
    _process_defaults(G, [action_parameters, runner_parameters])
    _validate(G)

    context = _resolve_dependencies(G)
    live_params = _cast_params_from(params, context, [action_parameters, runner_parameters])

    return live_params


def render_final_params(runner_parameters, action_parameters, params, action_context):
    '''
    Renders missing parameters required for action to execute. Treats parameters from the dict as
    plain values instead of trying to render them again. Returns dicts for action and runner
    parameters.
    '''
    G = _create_graph(action_context)

    # by that point, all params should already be resolved so any template should be treated value
    [G.add_node(name, value=value) for name, value in six.iteritems(params)]
    _process_defaults(G, [action_parameters, runner_parameters])
    _validate(G)

    context = _resolve_dependencies(G)
    context = _cast_params_from(context, context, [action_parameters, runner_parameters])

    return _split_params(runner_parameters, action_parameters, context)


def get_finalized_params(runnertype_parameter_info, action_parameter_info, liveaction_parameters,
                         action_context):
    '''
    Left here to keep tests running. Later we would need to split tests so they start testing each
    function separately.
    '''
    params = render_live_params(runnertype_parameter_info, action_parameter_info,
                                liveaction_parameters, action_context)
    return render_final_params(runnertype_parameter_info, action_parameter_info, params,
                               action_context)


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			import six
17			import networkx as nx
18
19			from jinja2 import meta
20			from st2common import log as logging
21			from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
22			from st2common.constants.keyvalue import SYSTEM_SCOPE
23			from st2common.exceptions.param import ParamException
24			from st2common.services.keyvalues import KeyValueLookup
25			from st2common.util.casts import get_cast
26			from st2common.util.compat import to_unicode
27			from st2common.util import jinja as jinja_utils
28
29
30			LOG = logging.getLogger(__name__)
31			ENV = jinja_utils.get_jinja_environment()
32
33			__all__ = [
34			'render_live_params',
35			'render_final_params',
36			]
37
38
39			def _split_params(runner_parameters, action_parameters, mixed_params):
40			def pf(params, skips):
41			result = {k: v for k, v in six.iteritems(mixed_params)
42			if k in params and k not in skips}
43			return result
44			return (pf(runner_parameters, {}), pf(action_parameters, runner_parameters))
45
46
47			def _cast_params(rendered, parameter_schemas):
48			'''
49			It's just here to make tests happy
50			'''
51			casted_params = {}
52			for k, v in six.iteritems(rendered):
53			casted_params[k] = _cast(v, parameter_schemas[k] or {})
54			return casted_params
55
56
57			def _cast(v, parameter_schema):
58			if v is None or not parameter_schema:
59			return v
60
61			parameter_type = parameter_schema.get('type', None)
62			if not parameter_type:
63			return v
64
65			cast = get_cast(cast_type=parameter_type)
66			if not cast:
67			return v
68
69			return cast(v)
70
71
72			def _create_graph(action_context):
73			'''
74			Creates a generic directed graph for depencency tree and fills it with basic context variables
75			'''
76			G = nx.DiGraph()
77			G.add_node(SYSTEM_SCOPE, value=KeyValueLookup(scope=SYSTEM_SCOPE))
78			G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
79			return G
80
81
82			def _process(G, name, value):
83			'''
84			Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
85			'''
86			# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
87			# Instead we're just assuming every string to be a unicode string
88			if isinstance(value, str):
89			value = to_unicode(value)
90
91			complex_value_str = None
92			if isinstance(value, list) or isinstance(value, dict):
93			complex_value_str = str(value)
94
95			is_jinja_expr = (
96			jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
97			complex_value_str
98			)
99			)
100
101			if is_jinja_expr:
102			G.add_node(name, template=value)
103
104			template_ast = ENV.parse(value)
105			LOG.debug('Template ast: %s', template_ast)
106			# Dependencies of the node represent jinja variables used in the template
107			# We're connecting nodes with an edge for every depencency to traverse them
108			# in the right order and also make sure that we don't have missing or cyclic
109			# dependencies upfront.
110			dependencies = meta.find_undeclared_variables(template_ast)
111			LOG.debug('Dependencies: %s', dependencies)
112			if dependencies:
113			for dependency in dependencies:
114			G.add_edge(dependency, name)
115			else:
116			G.add_node(name, value=value)
117
118
119			def _process_defaults(G, schemas):
120			'''
121			Process dependencies for parameters default values in the order schemas are defined.
122			'''
123			for schema in schemas:
124			for name, value in six.iteritems(schema):
125			absent = name not in G.node
126			is_none = G.node.get(name, {}).get('value') is None
127			immutable = value.get('immutable', False)
128			if absent or is_none or immutable:
129			_process(G, name, value.get('default'))
130
131
132			def _validate(G):
133			'''
134			Validates dependency graph to ensure it has no missing or cyclic dependencies
135			'''
136			for name in G.nodes():
137			if 'value' not in G.node[name] and 'template' not in G.node[name]:
138			msg = 'Dependecy unsatisfied in %s' % name
139			raise ParamException(msg)
140
141			if not nx.is_directed_acyclic_graph(G):
142			msg = 'Cyclic dependecy found'
143			raise ParamException(msg)
144
145
146			def _render(node, render_context):
147			'''
148			Render the node depending on its type
149			'''
150			if 'template' in node:
151			LOG.debug('Rendering node: %s with context: %s', node, render_context)
152			return ENV.from_string(node['template']).render(render_context)
153			if 'value' in node:
154			return node['value']
155
156
157			def _resolve_dependencies(G):
158			'''
159			Traverse the dependency graph starting from resolved nodes
160			'''
161			context = {}
162			for name in nx.topological_sort(G):
163			node = G.node[name]
164			try:
165			template = node.get('template', None)
166
167			# Special case for non simple types which contains Jinja notation (lists, dicts)
168			if 'template' in node and isinstance(template, (list, dict)):
169			if isinstance(template, list):
170			rendered_list = list()
171
172			for template in G.node[name]['template']:
173			rendered_list.append(
174			_render(dict(template=template), context)
175			)
176			context[name] = rendered_list
177			elif isinstance(template, dict):
178			rendered_dict = dict()
179
180			for key, value in G.node[name]['template'].items():
181			value = _render(dict(template=value), context)
182			rendered_dict[key] = value
183
184			context[name] = rendered_dict
185			else:
186			context[name] = _render(node, context)
187			except Exception as e:
188			LOG.debug('Failed to render %s: %s', name, e, exc_info=True)
189			msg = 'Failed to render parameter "%s": %s' % (name, str(e))
190			raise ParamException(msg)
191
192			return context
193
194
195			def _cast_params_from(params, context, schemas):
196			'''
197			Pick a list of parameters from context and cast each of them according to the schemas provided
198			'''
199			result = {}
200			for name in params:
201			param_schema = {}
202			for schema in schemas:
203			if name in schema:
204			param_schema = schema[name]
205			result[name] = _cast(context[name], param_schema)
206			return result
207
208
209			def render_live_params(runner_parameters, action_parameters, params, action_context):
210			'''
211			Renders list of parameters. Ensures that there's no cyclic or missing dependencies. Returns a
212			dict of plain rendered parameters.
213			'''
214			G = _create_graph(action_context)
215
216			[_process(G, name, value) for name, value in six.iteritems(params)]
217			_process_defaults(G, [action_parameters, runner_parameters])
218			_validate(G)
219
220			context = _resolve_dependencies(G)
221			live_params = _cast_params_from(params, context, [action_parameters, runner_parameters])
222
223			return live_params
224
225
226			def render_final_params(runner_parameters, action_parameters, params, action_context):
227			'''
228			Renders missing parameters required for action to execute. Treats parameters from the dict as
229			plain values instead of trying to render them again. Returns dicts for action and runner
230			parameters.
231			'''
232			G = _create_graph(action_context)
233
234			# by that point, all params should already be resolved so any template should be treated value
235			[G.add_node(name, value=value) for name, value in six.iteritems(params)]
236			_process_defaults(G, [action_parameters, runner_parameters])
237			_validate(G)
238
239			context = _resolve_dependencies(G)
240			context = _cast_params_from(context, context, [action_parameters, runner_parameters])
241
242			return _split_params(runner_parameters, action_parameters, context)
243
244
245			def get_finalized_params(runnertype_parameter_info, action_parameter_info, liveaction_parameters,
246			action_context):
247			'''
248			Left here to keep tests running. Later we would need to split tests so they start testing each
249			function separately.
250			'''
251			params = render_live_params(runnertype_parameter_info, action_parameter_info,
252			liveaction_parameters, action_context)
253			return render_final_params(runnertype_parameter_info, action_parameter_info, params,
254			action_context)
255

StackStorm / st2

Push — master ( 07d26d...59f2fc )

_resolve_dependencies() F

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like