provisions.base.provision() - Code Metrics - isomeric/isomer - Measure and Improve Code Quality continuously with Scrutinizer

provisions.base.provision() F
last analyzed 2021-02-24 12:38 UTC

↳ Parent: provisions.base

Complexity

Conditions

Size

Total Lines	100
Code Lines	71

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	17
eloc	71
nop	5
dl	0
loc	100
rs	1.8
c	0
b	0
f	0

How to fix Long Method Complexity

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Isomer - The distributed application framework
# ==============================================
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

Provisioning: Basic Functionality
=================================

Contains
--------

Basic functionality around provisioning.


"""

from networkx import DiGraph, is_directed_acyclic_graph, simple_cycles
from networkx.algorithms import topological_sort
from jsonschema import ValidationError

from isomer.logger import isolog, debug, verbose, warn, error


def log(*args, **kwargs):
    """Log as Emitter:MANAGE"""

    kwargs.update({"emitter": "PROVISIONS", "frame_ref": 2})
    isolog(*args, **kwargs)


def provisionList(
    items, database_name, overwrite=False, clear=False, skip_user_check=False
):
    """Provisions a list of items according to their schema

    :param items: A list of provisionable items.
    :param database_name:
    :param overwrite: Causes existing items to be overwritten
    :param clear: Clears the collection first (Danger!)
    :param skip_user_check: Skips checking if a system user is existing already
           (for user provisioning)
    :return:
    """

    log("Provisioning", items, database_name, lvl=debug)

    def get_system_user():
        """Retrieves the node local system user"""

        user = objectmodels["user"].find_one({"name": "System"})

        try:
            log("System user uuid: ", user.uuid, lvl=verbose)
            return user.uuid
        except AttributeError as system_user_error:
            log("No system user found:", system_user_error, lvl=warn)
            log(
                "Please install the user provision to setup a system user or "
                "check your database configuration",
                lvl=error,
            )
            return False

    # TODO: Do not check this on specific objects but on the model (i.e. once)
    def needs_owner(obj):
        """Determines whether a basic object has an ownership field"""
        for privilege in obj._fields.get("perms", None):
            if "owner" in obj._fields["perms"][privilege]:
                return True

        return False

    import pymongo
    from isomer.database import objectmodels, dbhost, dbport, dbname

    database_object = objectmodels[database_name]

    log(dbhost, dbname)
    # TODO: Fix this to make use of the dbhost

    client = pymongo.MongoClient(dbhost, dbport)
    db = client[dbname]

    if not skip_user_check:
        system_user = get_system_user()

        if not system_user:
            return
    else:
        # TODO: Evaluate what to do instead of using a hardcoded UUID
        # This is usually only here for provisioning the system user
        # One way to avoid this, is to create (instead of provision)
        # this one upon system installation.
        system_user = "0ba87daa-d315-462e-9f2e-6091d768fd36"

    col_name = database_object.collection_name()

    if clear is True:
        log("Clearing collection for", col_name, lvl=warn)
        db.drop_collection(col_name)
    counter = 0

    for no, item in enumerate(items):
        new_object = None
        item_uuid = item["uuid"]
        log("Validating object (%i/%i):" % (no + 1, len(items)), item_uuid, lvl=debug)

        if database_object.count({"uuid": item_uuid}) > 0:
            log("Object already present", lvl=warn)
            if overwrite is False:
                log("Not updating item", item, lvl=warn)
            else:
                log("Overwriting item: ", item_uuid, lvl=warn)
                new_object = database_object.find_one({"uuid": item_uuid})
                new_object._fields.update(item)
        else:
            new_object = database_object(item)

        if new_object is not None:
            try:
                if needs_owner(new_object):
                    if not hasattr(new_object, "owner"):
                        log("Adding system owner to object.", lvl=verbose)
                        new_object.owner = system_user
            except Exception as e:
                log("Error during ownership test:", e, type(e), exc=True, lvl=error)
            try:
                new_object.validate()
                new_object.save()
                counter += 1
            except ValidationError as e:
                raise ValidationError(
                    "Could not provision object: " + str(item_uuid), e
                )

    log("Provisioned %i out of %i items successfully." % (counter, len(items)))


def provision(
    list_provisions=False,
    overwrite=False,
    clear_provisions=False,
    package=None,
    installed=None,
):
    from isomer.provisions import build_provision_store
    from isomer.database import objectmodels

    provision_store = build_provision_store()

    if installed is None:
        installed = []

    def sort_dependencies(items):
        """Topologically sort the dependency tree"""

        g = DiGraph()
        log("Sorting dependencies")

        for key, item in items:
            log("key: ", key, "item:", item, pretty=True, lvl=debug)
            dependencies = item.get("dependencies", [])
            if isinstance(dependencies, str):
                dependencies = [dependencies]

            if key not in g:
                g.add_node(key)

            for link in dependencies:
                g.add_edge(key, link)

        if not is_directed_acyclic_graph(g):
            log("Cycles in provisioning dependency graph detected!", lvl=error)
            log("Involved provisions:", list(simple_cycles(g)), lvl=error)

        topology = list(topological_sort(g))
        topology.reverse()
        topology = list(set(topology).difference(installed))

        # log(topology, pretty=True)

        return topology

    sorted_provisions = sort_dependencies(provision_store.items())

    # These need to be installed first in that order:
    if "system" in sorted_provisions:
        sorted_provisions.remove("system")
    if "user" in sorted_provisions:
        sorted_provisions.remove("user")
    if "system" not in installed:
        sorted_provisions.insert(0, "system")
    if "user" not in installed:
        sorted_provisions.insert(0, "user")

    if list_provisions:
        log(sorted_provisions, pretty=True)
        exit()

    def provision_item(provision_name):
        """Provision a single provisioning element"""

        item = provision_store[provision_name]

        method = item.get("method", provisionList)
        model = item.get("model")
        data = item.get("data")

        method(data, model, overwrite=overwrite, clear=clear_provisions)

        confirm_provision(provision_name)

    def confirm_provision(provision_name):
        if provision_name == "user":
            log("Not confirming system user provision")
            return
        systemconfig = objectmodels["systemconfig"].find_one({"active": True})
        if provision_name not in systemconfig.provisions["packages"]:
            systemconfig.provisions["packages"].append(provision_name)
            systemconfig.save()

    if package is not None:
        if package in provision_store:
            log("Provisioning ", package, pretty=True)
            provision_item(package)
        else:
            log(
                "Unknown package: ",
                package,
                "\nValid provisionable packages are",
                list(provision_store.keys()),
                lvl=error,
                emitter="MANAGE",
            )
    else:
        for name in sorted_provisions:
            log("Provisioning", name, pretty=True)
            provision_item(name)


1			#!/usr/bin/env python
2			# -- coding: UTF-8 --
3
4			# Isomer - The distributed application framework
5			# ==============================================
6			# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
7			#
8			# This program is free software: you can redistribute it and/or modify
9			# it under the terms of the GNU Affero General Public License as published by
10			# the Free Software Foundation, either version 3 of the License, or
11			# (at your option) any later version.
12			#
13			# This program is distributed in the hope that it will be useful,
14			# but WITHOUT ANY WARRANTY; without even the implied warranty of
15			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16			# GNU Affero General Public License for more details.
17			#
18			# You should have received a copy of the GNU Affero General Public License
19			# along with this program. If not, see <http://www.gnu.org/licenses/>.
20
21			"""
22
23			Provisioning: Basic Functionality
24			=================================
25
26			Contains
27			--------
28
29			Basic functionality around provisioning.
30
31
32			"""
33
34			from networkx import DiGraph, is_directed_acyclic_graph, simple_cycles
35			from networkx.algorithms import topological_sort
36			from jsonschema import ValidationError
37
38			from isomer.logger import isolog, debug, verbose, warn, error
39
40
41			def log(args, *kwargs):
42			"""Log as Emitter:MANAGE"""
43
44			kwargs.update({"emitter": "PROVISIONS", "frame_ref": 2})
45			isolog(args, *kwargs)
46
47
48			def provisionList(
49			items, database_name, overwrite=False, clear=False, skip_user_check=False
50			):
51			"""Provisions a list of items according to their schema
52
53			:param items: A list of provisionable items.
54			:param database_name:
55			:param overwrite: Causes existing items to be overwritten
56			:param clear: Clears the collection first (Danger!)
57			:param skip_user_check: Skips checking if a system user is existing already
58			(for user provisioning)
59			:return:
60			"""
61
62			log("Provisioning", items, database_name, lvl=debug)
63
64			def get_system_user():
65			"""Retrieves the node local system user"""
66
67			user = objectmodels["user"].find_one({"name": "System"})
68
69			try:
70			log("System user uuid: ", user.uuid, lvl=verbose)
71			return user.uuid
72			except AttributeError as system_user_error:
73			log("No system user found:", system_user_error, lvl=warn)
74			log(
75			"Please install the user provision to setup a system user or "
76			"check your database configuration",
77			lvl=error,
78			)
79			return False
80
81			# TODO: Do not check this on specific objects but on the model (i.e. once)
82			def needs_owner(obj):
83			"""Determines whether a basic object has an ownership field"""
84			for privilege in obj._fields.get("perms", None):
85			if "owner" in obj._fields["perms"][privilege]:
86			return True
87
88			return False
89
90			import pymongo
91			from isomer.database import objectmodels, dbhost, dbport, dbname
92
93			database_object = objectmodels[database_name]
94
95			log(dbhost, dbname)
96			# TODO: Fix this to make use of the dbhost
97
98			client = pymongo.MongoClient(dbhost, dbport)
99			db = client[dbname]
100
101			if not skip_user_check:
102			system_user = get_system_user()
103
104			if not system_user:
105			return
106			else:
107			# TODO: Evaluate what to do instead of using a hardcoded UUID
108			# This is usually only here for provisioning the system user
109			# One way to avoid this, is to create (instead of provision)
110			# this one upon system installation.
111			system_user = "0ba87daa-d315-462e-9f2e-6091d768fd36"
112
113			col_name = database_object.collection_name()
114
115			if clear is True:
116			log("Clearing collection for", col_name, lvl=warn)
117			db.drop_collection(col_name)
118			counter = 0
119
120			for no, item in enumerate(items):
121			new_object = None
122			item_uuid = item["uuid"]
123			log("Validating object (%i/%i):" % (no + 1, len(items)), item_uuid, lvl=debug)
124
125			if database_object.count({"uuid": item_uuid}) > 0:
126			log("Object already present", lvl=warn)
127			if overwrite is False:
128			log("Not updating item", item, lvl=warn)
129			else:
130			log("Overwriting item: ", item_uuid, lvl=warn)
131			new_object = database_object.find_one({"uuid": item_uuid})
132			new_object._fields.update(item)
133			else:
134			new_object = database_object(item)
135
136			if new_object is not None:
137			try:
138			if needs_owner(new_object):
139			if not hasattr(new_object, "owner"):
140			log("Adding system owner to object.", lvl=verbose)
141			new_object.owner = system_user
142			except Exception as e:
143			log("Error during ownership test:", e, type(e), exc=True, lvl=error)
144			try:
145			new_object.validate()
146			new_object.save()
147			counter += 1
148			except ValidationError as e:
149			raise ValidationError(
150			"Could not provision object: " + str(item_uuid), e
151			)
152
153			log("Provisioned %i out of %i items successfully." % (counter, len(items)))
154
155
156			def provision(
157			list_provisions=False,
158			overwrite=False,
159			clear_provisions=False,
160			package=None,
161			installed=None,
162			):
163			from isomer.provisions import build_provision_store
164			from isomer.database import objectmodels
165
166			provision_store = build_provision_store()
167
168			if installed is None:
169			installed = []
170
171			def sort_dependencies(items):
172			"""Topologically sort the dependency tree"""
173
174			g = DiGraph()
175			log("Sorting dependencies")
176
177			for key, item in items:
178			log("key: ", key, "item:", item, pretty=True, lvl=debug)
179			dependencies = item.get("dependencies", [])
180			if isinstance(dependencies, str):
181			dependencies = [dependencies]
182
183			if key not in g:
184			g.add_node(key)
185
186			for link in dependencies:
187			g.add_edge(key, link)
188
189			if not is_directed_acyclic_graph(g):
190			log("Cycles in provisioning dependency graph detected!", lvl=error)
191			log("Involved provisions:", list(simple_cycles(g)), lvl=error)
192
193			topology = list(topological_sort(g))
194			topology.reverse()
195			topology = list(set(topology).difference(installed))
196
197			# log(topology, pretty=True)
198
199			return topology
200
201			sorted_provisions = sort_dependencies(provision_store.items())
202
203			# These need to be installed first in that order:
204			if "system" in sorted_provisions:
205			sorted_provisions.remove("system")
206			if "user" in sorted_provisions:
207			sorted_provisions.remove("user")
208			if "system" not in installed:
209			sorted_provisions.insert(0, "system")
210			if "user" not in installed:
211			sorted_provisions.insert(0, "user")
212
213			if list_provisions:
214			log(sorted_provisions, pretty=True)
215			exit()
216
217			def provision_item(provision_name):
218			"""Provision a single provisioning element"""
219
220			item = provision_store[provision_name]
221
222			method = item.get("method", provisionList)
223			model = item.get("model")
224			data = item.get("data")
225
226			method(data, model, overwrite=overwrite, clear=clear_provisions)
227
228			confirm_provision(provision_name)
229
230			def confirm_provision(provision_name):
231			if provision_name == "user":
232			log("Not confirming system user provision")
233			return
234			systemconfig = objectmodels["systemconfig"].find_one({"active": True})
235			if provision_name not in systemconfig.provisions["packages"]:
236			systemconfig.provisions["packages"].append(provision_name)
237			systemconfig.save()
238
239			if package is not None:
240			if package in provision_store:
241			log("Provisioning ", package, pretty=True)
242			provision_item(package)
243			else:
244			log(
245			"Unknown package: ",
246			package,
247			"\nValid provisionable packages are",
248			list(provision_store.keys()),
249			lvl=error,
250			emitter="MANAGE",
251			)
252			else:
253			for name in sorted_provisions:
254			log("Provisioning", name, pretty=True)
255			provision_item(name)
256

isomeric / isomer

provisions.base.provision() F last analyzed 2021-02-24 12:38 UTC

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like

provisions.base.provision() F
last analyzed 2021-02-24 12:38 UTC