isomer.database._build_collections() - Code Metrics - Inspection of "removes (hopefully last ;) moved HFOS content" - isomeric/isomer - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Push — master ( 1c66f3...c17ee3 )

by Heiko 'riot'

created 2020-12-20 19:23 UTC

isomer.database._build_collections() C

↳ Parent: isomer.database

Complexity

Conditions

Size

Total Lines	80
Code Lines	59

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	11
eloc	59
nop	1
dl	0
loc	80
rs	5.1218
c	0
b	0
f	0

How to fix Long Method Complexity

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Isomer - The distributed application framework
# ==============================================
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""


Module: Database
================

Contains the underlying object model manager and generates object factories
from schemata.

Contains
========

Objectstore builder functions.


"""

import sys
import formal
import jsonschema
import pymongo

from isomer import schemastore
from isomer.error import abort, EXIT_NO_DATABASE, EXIT_NO_DATABASE_DEFINED,\
    EXIT_NO_DATABASE_HOST
from isomer.logger import isolog, warn, critical, debug, verbose, error
from isomer.misc.std import std_color


def db_log(*args, **kwargs):
    """Log as emitter 'DB'"""
    kwargs.update({"emitter": "DB", "frame_ref": 2})
    isolog(*args, **kwargs)


objectmodels = None
collections = None

dbhost = ""
dbport = 0
dbname = ""
instance = ""
initialized = False
ValidationError = jsonschema.ValidationError


def clear_all():
    """DANGER!
    *This command is a maintenance tool and clears the complete database.*
    """

    sure = input(
        "Are you sure to drop the complete database content? (Type "
        "in upppercase YES)"
    )
    if not (sure == "YES"):
        db_log("Not deleting the database.")
        sys.exit()

    client = pymongo.MongoClient(host=dbhost, port=dbport)
    db = client[dbname]

    for col in db.collection_names(include_system_collections=False):
        db_log("Dropping collection ", col, lvl=warn)
        db.drop_collection(col)


class IsomerBaseModel(formal.formalModel):
    """Base Isomer Dataclass"""

    def save(self, *args, **kwargs):
        """Set a random default color"""
        if self._fields.get("color", None) is None:
            self._fields["color"] = std_color()
        super(IsomerBaseModel, self).save(*args, **kwargs)

    @classmethod
    def by_uuid(cls, uuid):
        """Find data object by uuid"""
        return cls.find_one({"uuid": uuid})


def _build_model_factories(store):
    """Generate factories to construct objects from schemata"""

    result = {}

    for schemaname in store:

        schema = None

        try:
            schema = store[schemaname]["schema"]
        except KeyError:
            db_log("No schema found for ", schemaname, lvl=critical, exc=True)

        try:
            result[schemaname] = formal.model_factory(schema, IsomerBaseModel)
        except Exception as e:
            db_log(
                "Could not create factory for schema ",
                schemaname,
                schema,
                lvl=critical,
                exc=True,
            )

    return result


def _build_collections(store):
    """Generate database collections with indices from the schemastore"""

    result = {}

    client = pymongo.MongoClient(host=dbhost, port=dbport)
    db = client[dbname]

    for schemaname in store:

        schema = None
        indices = None

        try:
            schema = store[schemaname]["schema"]
            indices = store[schemaname].get("indices", None)
        except KeyError:
            db_log("No schema found for ", schemaname, lvl=critical)

        try:
            result[schemaname] = db[schemaname]
        except Exception:
            db_log(
                "Could not get collection for schema ",
                schemaname,
                schema,
                lvl=critical,
                exc=True,
            )

        if indices is None:
            continue

        col = db[schemaname]
        db_log("Adding indices to", schemaname, lvl=debug)
        i = 0
        keys = list(indices.keys())

        while i < len(indices):
            index_name = keys[i]
            index = indices[index_name]

            index_type = index.get("type", None)
            index_unique = index.get("unique", False)
            index_sparse = index.get("sparse", True)
            index_reindex = index.get("reindex", False)

            if index_type in (None, "text"):
                index_type = pymongo.TEXT
            elif index_type == "2dsphere":
                index_type = pymongo.GEOSPHERE

            def do_index():
                """Ensure index on a data class"""
                col.ensure_index(
                    [(index_name, index_type)], unique=index_unique, sparse=index_sparse
                )

            db_log("Enabling index of type", index_type, "on", index_name, lvl=debug)
            try:
                do_index()
                i += 1
            except pymongo.errors.OperationFailure:
                db_log(col.list_indexes().__dict__, pretty=True, lvl=verbose)
                if not index_reindex:
                    db_log("Index was not created!", lvl=warn)
                    i += 1
                else:
                    try:
                        col.drop_index(index_name)
                        do_index()
                        i += 1
                    except pymongo.errors.OperationFailure as e:
                        db_log("Index recreation problem:", exc=True, lvl=error)
                        col.drop_indexes()
                        i = 0

                        # for index in col.list_indexes():
                        #    db_log("Index: ", index)
    return result


def initialize(
    address="127.0.0.1:27017",
    database_name="isomer-default",
    instance_name="default",
    reload=False,
    ignore_fail=False,
):
    """Initializes the database connectivity, schemata and finally object models"""

    global objectmodels
    global collections
    global dbhost
    global dbport
    global dbname
    global instance
    global initialized

    if initialized and not reload:
        isolog(
            "Already initialized and not reloading.",
            lvl=warn,
            emitter="DB",
            frame_ref=2,
        )
        return

    dbhost = address.split(":")[0]
    dbport = int(address.split(":")[1]) if ":" in address else 27017
    dbname = database_name

    db_log("Using database:", dbname, "@", dbhost, ":", dbport)

    if dbname == "" and not ignore_fail:
        abort(EXIT_NO_DATABASE_DEFINED)
    if dbhost == "" and not ignore_fail:
        abort(EXIT_NO_DATABASE_HOST)

    try:
        client = pymongo.MongoClient(host=dbhost, port=dbport)
        db = client[dbname]
        db_log("Database: ", db.command("buildinfo"), lvl=debug)
    except Exception as e:
        log_level = warn if ignore_fail else critical
        db_log(
            "No database available! Check if you have mongodb > 2.2 "
            "installed and running as well as listening on port %i "
            "of %s and check if you specified the correct "
            "instance and environment. Detailed Error:\n%s" % (dbport, dbhost, e),
            lvl=log_level,
        )
        if not ignore_fail:
            abort(EXIT_NO_DATABASE)
        else:
            return False

    formal.connect(database_name, host=dbhost, port=dbport)
    formal.connect_sql(database_name, database_type="sql_memory")

    schemastore.schemastore = schemastore.build_schemastore_new()
    schemastore.l10n_schemastore = schemastore.build_l10n_schemastore(
        schemastore.schemastore
    )
    objectmodels = _build_model_factories(schemastore.schemastore)
    collections = _build_collections(schemastore.schemastore)
    instance = instance_name
    initialized = True

    return True


1			#!/usr/bin/env python
2			# -- coding: UTF-8 --
3
4			# Isomer - The distributed application framework
5			# ==============================================
6			# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
7			#
8			# This program is free software: you can redistribute it and/or modify
9			# it under the terms of the GNU Affero General Public License as published by
10			# the Free Software Foundation, either version 3 of the License, or
11			# (at your option) any later version.
12			#
13			# This program is distributed in the hope that it will be useful,
14			# but WITHOUT ANY WARRANTY; without even the implied warranty of
15			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16			# GNU Affero General Public License for more details.
17			#
18			# You should have received a copy of the GNU Affero General Public License
19			# along with this program. If not, see <http://www.gnu.org/licenses/>.
20
21			"""
22
23
24			Module: Database
25			================
26
27			Contains the underlying object model manager and generates object factories
28			from schemata.
29
30			Contains
31			========
32
33			Objectstore builder functions.
34
35
36			"""
37
38			import sys
39			import formal
40			import jsonschema
41			import pymongo
42
43			from isomer import schemastore
44			from isomer.error import abort, EXIT_NO_DATABASE, EXIT_NO_DATABASE_DEFINED,\
45			EXIT_NO_DATABASE_HOST
46			from isomer.logger import isolog, warn, critical, debug, verbose, error
47			from isomer.misc.std import std_color
48
49
50			def db_log(args, *kwargs):
51			"""Log as emitter 'DB'"""
52			kwargs.update({"emitter": "DB", "frame_ref": 2})
53			isolog(args, *kwargs)
54
55
56			objectmodels = None
57			collections = None
58
59			dbhost = ""
60			dbport = 0
61			dbname = ""
62			instance = ""
63			initialized = False
64			ValidationError = jsonschema.ValidationError
65
66
67			def clear_all():
68			"""DANGER!
69			This command is a maintenance tool and clears the complete database.
70			"""
71
72			sure = input(
73			"Are you sure to drop the complete database content? (Type "
74			"in upppercase YES)"
75			)
76			if not (sure == "YES"):
77			db_log("Not deleting the database.")
78			sys.exit()
79
80			client = pymongo.MongoClient(host=dbhost, port=dbport)
81			db = client[dbname]
82
83			for col in db.collection_names(include_system_collections=False):
84			db_log("Dropping collection ", col, lvl=warn)
85			db.drop_collection(col)
86
87
88			class IsomerBaseModel(formal.formalModel):
89			"""Base Isomer Dataclass"""
90
91			def save(self, args, *kwargs):
92			"""Set a random default color"""
93			if self._fields.get("color", None) is None:
94			self._fields["color"] = std_color()
95			super(IsomerBaseModel, self).save(args, *kwargs)
96
97			@classmethod
98			def by_uuid(cls, uuid):
99			"""Find data object by uuid"""
100			return cls.find_one({"uuid": uuid})
101
102
103			def _build_model_factories(store):
104			"""Generate factories to construct objects from schemata"""
105
106			result = {}
107
108			for schemaname in store:
109
110			schema = None
111
112			try:
113			schema = store[schemaname]["schema"]
114			except KeyError:
115			db_log("No schema found for ", schemaname, lvl=critical, exc=True)
116
117			try:
118			result[schemaname] = formal.model_factory(schema, IsomerBaseModel)
119			except Exception as e:
120			db_log(
121			"Could not create factory for schema ",
122			schemaname,
123			schema,
124			lvl=critical,
125			exc=True,
126			)
127
128			return result
129
130
131			def _build_collections(store):
132			"""Generate database collections with indices from the schemastore"""
133
134			result = {}
135
136			client = pymongo.MongoClient(host=dbhost, port=dbport)
137			db = client[dbname]
138
139			for schemaname in store:
140
141			schema = None
142			indices = None
143
144			try:
145			schema = store[schemaname]["schema"]
146			indices = store[schemaname].get("indices", None)
147			except KeyError:
148			db_log("No schema found for ", schemaname, lvl=critical)
149
150			try:
151			result[schemaname] = db[schemaname]
152			except Exception:
153			db_log(
154			"Could not get collection for schema ",
155			schemaname,
156			schema,
157			lvl=critical,
158			exc=True,
159			)
160
161			if indices is None:
162			continue
163
164			col = db[schemaname]
165			db_log("Adding indices to", schemaname, lvl=debug)
166			i = 0
167			keys = list(indices.keys())
168
169			while i < len(indices):
170			index_name = keys[i]
171			index = indices[index_name]
172
173			index_type = index.get("type", None)
174			index_unique = index.get("unique", False)
175			index_sparse = index.get("sparse", True)
176			index_reindex = index.get("reindex", False)
177
178			if index_type in (None, "text"):
179			index_type = pymongo.TEXT
180			elif index_type == "2dsphere":
181			index_type = pymongo.GEOSPHERE
182
183			def do_index():
184			"""Ensure index on a data class"""
185			col.ensure_index(
186			[(index_name, index_type)], unique=index_unique, sparse=index_sparse
187			)
188
189			db_log("Enabling index of type", index_type, "on", index_name, lvl=debug)
190			try:
191			do_index()
192			i += 1
193			except pymongo.errors.OperationFailure:
194			db_log(col.list_indexes().__dict__, pretty=True, lvl=verbose)
195			if not index_reindex:
196			db_log("Index was not created!", lvl=warn)
197			i += 1
198			else:
199			try:
200			col.drop_index(index_name)
201			do_index()
202			i += 1
203			except pymongo.errors.OperationFailure as e:
204			db_log("Index recreation problem:", exc=True, lvl=error)
205			col.drop_indexes()
206			i = 0
207
208			# for index in col.list_indexes():
209			# db_log("Index: ", index)
210			return result
211
212
213			def initialize(
214			address="127.0.0.1:27017",
215			database_name="isomer-default",
216			instance_name="default",
217			reload=False,
218			ignore_fail=False,
219			):
220			"""Initializes the database connectivity, schemata and finally object models"""
221
222			global objectmodels
223			global collections
224			global dbhost
225			global dbport
226			global dbname
227			global instance
228			global initialized
229
230			if initialized and not reload:
231			isolog(
232			"Already initialized and not reloading.",
233			lvl=warn,
234			emitter="DB",
235			frame_ref=2,
236			)
237			return
238
239			dbhost = address.split(":")[0]
240			dbport = int(address.split(":")[1]) if ":" in address else 27017
241			dbname = database_name
242
243			db_log("Using database:", dbname, "@", dbhost, ":", dbport)
244
245			if dbname == "" and not ignore_fail:
246			abort(EXIT_NO_DATABASE_DEFINED)
247			if dbhost == "" and not ignore_fail:
248			abort(EXIT_NO_DATABASE_HOST)
249
250			try:
251			client = pymongo.MongoClient(host=dbhost, port=dbport)
252			db = client[dbname]
253			db_log("Database: ", db.command("buildinfo"), lvl=debug)
254			except Exception as e:
255			log_level = warn if ignore_fail else critical
256			db_log(
257			"No database available! Check if you have mongodb > 2.2 "
258			"installed and running as well as listening on port %i "
259			"of %s and check if you specified the correct "
260			"instance and environment. Detailed Error:\n%s" % (dbport, dbhost, e),
261			lvl=log_level,
262			)
263			if not ignore_fail:
264			abort(EXIT_NO_DATABASE)
265			else:
266			return False
267
268			formal.connect(database_name, host=dbhost, port=dbport)
269			formal.connect_sql(database_name, database_type="sql_memory")
270
271			schemastore.schemastore = schemastore.build_schemastore_new()
272			schemastore.l10n_schemastore = schemastore.build_l10n_schemastore(
273			schemastore.schemastore
274			)
275			objectmodels = _build_model_factories(schemastore.schemastore)
276			collections = _build_collections(schemastore.schemastore)
277			instance = instance_name
278			initialized = True
279
280			return True
281

isomeric / isomer

Push — master ( 1c66f3...c17ee3 )

isomer.database._build_collections() C

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like