print_load_avg() - Code Metrics - Inspection of "Follow Bintray repo naming for Centos/RHEL builds" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2304)

by Arma

created 2015-12-17 19:15 UTC

print_load_avg() F

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
dl	0
loc	36
rs	3.1765
cc	11

How to fix Complexity

#!/usr/bin/env python
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""

Tags: Ops tool.

A utility script that diffs models registered in st2 db versus what's on disk.

"""

import difflib
import json
import os
import sys

import eventlet
from oslo_config import cfg

from st2common import config
from st2common.constants.pack import DEFAULT_PACK_NAME
from st2common.content.loader import ContentPackLoader
from st2common.content.loader import MetaLoader
from st2common.bootstrap.base import ResourceRegistrar
import st2common.content.utils as content_utils
from st2common.models.api.action import ActionAPI
from st2common.models.api.sensor import SensorTypeAPI
from st2common.models.api.rule import RuleAPI
from st2common.models.db import db_setup
from st2common.models.db import db_teardown
from st2common.models.system.common import ResourceReference
from st2common.persistence.rule import Rule
from st2common.persistence.sensor import SensorType
from st2common.persistence.action import Action

registrar = ResourceRegistrar()
registrar.ALLOWED_EXTENSIONS = ['.yaml', '.yml', '.json']

meta_loader = MetaLoader()

API_MODELS_ARTIFACT_TYPES = {
    'actions': ActionAPI,
    'sensors': SensorTypeAPI,
    'rules': RuleAPI
}

API_MODELS_PERSISTENT_MODELS = {
    Action: ActionAPI,
    SensorType: SensorTypeAPI,
    Rule: RuleAPI
}


def do_register_cli_opts(opts, ignore_errors=False):
    for opt in opts:
        try:
            cfg.CONF.register_cli_opt(opt)
        except:
            if not ignore_errors:
                raise


def _monkey_patch():
    eventlet.monkey_patch(
        os=True,
        select=True,
        socket=True,
        thread=False if '--use-debugger' in sys.argv else True,
        time=True)


def _get_api_models_from_db(persistence_model, pack_dir=None):
    filters = {}
    if pack_dir:
        pack_name = os.path.basename(os.path.normpath(pack_dir))
        filters = {'pack': pack_name}
    models = persistence_model.query(**filters)
    models_dict = {}
    for model in models:
        model_pack = getattr(model, 'pack', None) or DEFAULT_PACK_NAME
        model_ref = ResourceReference.to_string_reference(name=model.name, pack=model_pack)
        if getattr(model, 'id', None):
            del model.id
        API_MODEL = API_MODELS_PERSISTENT_MODELS[persistence_model]
        models_dict[model_ref] = API_MODEL.from_model(model)
    return models_dict


def _get_api_models_from_disk(artifact_type, pack_dir=None):
    loader = ContentPackLoader()
    artifacts = None

    if pack_dir:
        artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
        pack_name = os.path.basename(os.path.normpath(pack_dir))
        artifacts = {pack_name: artifacts_dir}
    else:
        packs_dirs = content_utils.get_packs_base_paths()
        artifacts = loader.get_content(packs_dirs, artifact_type)

    artifacts_dict = {}
    for pack_name, pack_path in artifacts.items():
        artifacts_paths = registrar.get_resources_from_pack(pack_path)
        for artifact_path in artifacts_paths:
            artifact = meta_loader.load(artifact_path)
            if artifact_type == 'sensors':
                sensors_dir = os.path.dirname(artifact_path)
                sensor_file_path = os.path.join(sensors_dir, artifact['entry_point'])
                artifact['artifact_uri'] = 'file://' + sensor_file_path
            name = artifact.get('name', None) or artifact.get('class_name', None)
            if not artifact.get('pack', None):
                artifact['pack'] = pack_name
            ref = ResourceReference.to_string_reference(name=name,
                                                        pack=pack_name)
            API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
            # Following conversions are required because we add some fields with
            # default values in db model. If we don't do these conversions,
            # we'll see a unnecessary diff for those fields.
            artifact_api = API_MODEL(**artifact)
            artifact_db = API_MODEL.to_model(artifact_api)
            artifact_api = API_MODEL.from_model(artifact_db)
            artifacts_dict[ref] = artifact_api

    return artifacts_dict


def _content_diff(artifact_type=None, artifact_in_disk=None, artifact_in_db=None,
                  verbose=False):
    artifact_in_disk_str = json.dumps(
        artifact_in_disk.__json__(), sort_keys=True,
        indent=4, separators=(',', ': ')
    )
    artifact_in_db_str = json.dumps(
        artifact_in_db.__json__(), sort_keys=True,
        indent=4, separators=(',', ': ')
    )
    diffs = difflib.context_diff(artifact_in_db_str.splitlines(),
                                 artifact_in_disk_str.splitlines(),
                                 fromfile='DB contents', tofile='Disk contents')
    printed = False
    for diff in diffs:
        if not printed:
            identifier = getattr(artifact_in_db, 'ref', getattr(artifact_in_db, 'name'))
            print('%s %s in db differs from what is in disk.' % (artifact_type.upper(),
                  identifier))
            printed = True
        print(diff)

    if verbose:
        print('\n\nOriginal contents:')
        print('===================\n')
        print('Artifact in db:\n\n%s\n\n' % artifact_in_db_str)
        print('Artifact in disk:\n\n%s\n\n' % artifact_in_disk_str)


def _diff(persistence_model, artifact_type, pack_dir=None, verbose=True,
          content_diff=True):
    artifacts_in_db_dict = _get_api_models_from_db(persistence_model, pack_dir=pack_dir)
    artifacts_in_disk_dict = _get_api_models_from_disk(artifact_type, pack_dir=pack_dir)

    # print(artifacts_in_disk_dict)
    all_artifacts = set(artifacts_in_db_dict.keys() + artifacts_in_disk_dict.keys())

    for artifact in all_artifacts:
        artifact_in_db = artifacts_in_db_dict.get(artifact, None)
        artifact_in_disk = artifacts_in_disk_dict.get(artifact, None)
        artifact_in_disk_pretty_json = None
        artifact_in_db_pretty_json = None

        if verbose:
            print('******************************************************************************')
            print('Checking if artifact %s is present in both disk and db.' % artifact)
        if not artifact_in_db:
            print('##############################################################################')
            print('%s %s in disk not available in db.' % (artifact_type.upper(), artifact))
            artifact_in_disk_pretty_json = json.dumps(
                artifact_in_disk.__json__(), sort_keys=True,
                indent=4, separators=(',', ': ')
            )
            if verbose:
                print('File contents: \n')
                print(artifact_in_disk_pretty_json)
            continue

        if not artifact_in_disk:
            print('##############################################################################')
            print('%s %s in db not available in disk.' % (artifact_type.upper(), artifact))
            artifact_in_db_pretty_json = json.dumps(
                artifact_in_db.__json__(), sort_keys=True,
                indent=4, separators=(',', ': ')
            )
            if verbose:
                print('DB contents: \n')
                print(artifact_in_db_pretty_json)
            continue
        if verbose:
            print('Artifact %s exists in both disk and db.' % artifact)

        if content_diff:
            if verbose:
                print('Performing content diff for artifact %s.' % artifact)

            _content_diff(artifact_type=artifact_type,
                          artifact_in_disk=artifact_in_disk,
                          artifact_in_db=artifact_in_db,
                          verbose=verbose)


def _diff_actions(pack_dir=None, verbose=False, content_diff=True):
    _diff(Action, 'actions', pack_dir=pack_dir,
          verbose=verbose, content_diff=content_diff)


def _diff_sensors(pack_dir=None, verbose=False, content_diff=True):
    _diff(SensorType, 'sensors', pack_dir=pack_dir,
          verbose=verbose, content_diff=content_diff)


def _diff_rules(pack_dir=None, verbose=True, content_diff=True):
    _diff(Rule, 'rules', pack_dir=pack_dir,
          verbose=verbose, content_diff=content_diff)


def main():
    _monkey_patch()

    cli_opts = [
        cfg.BoolOpt('sensors', default=False,
                    help='diff sensor alone.'),
        cfg.BoolOpt('actions', default=False,
                    help='diff actions alone.'),
        cfg.BoolOpt('rules', default=False,
                    help='diff rules alone.'),
        cfg.BoolOpt('all', default=False,
                    help='diff sensors, actions and rules.'),
        cfg.BoolOpt('verbose', default=False),
        cfg.BoolOpt('simple', default=False,
                    help='In simple mode, tool only tells you if content is missing.' +
                         'It doesn\'t show you content diff between disk and db.'),
        cfg.StrOpt('pack-dir', default=None, help='Path to specific pack to diff.')
    ]
    do_register_cli_opts(cli_opts)
    config.parse_args()

    username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
    password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None

    # Connect to db.
    db_setup(cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
             username=username, password=password)

    # Diff content
    pack_dir = cfg.CONF.pack_dir or None
    content_diff = not cfg.CONF.simple

    if cfg.CONF.all:
        _diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
        _diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
        _diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
        return

    if cfg.CONF.sensors:
        _diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)

    if cfg.CONF.actions:
        _diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)

    if cfg.CONF.rules:
        _diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)

    # Disconnect from db.
    db_teardown()

if __name__ == '__main__':
    main()


1			#!/usr/bin/env python
2			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
3			# contributor license agreements. See the NOTICE file distributed with
4			# this work for additional information regarding copyright ownership.
5			# The ASF licenses this file to You under the Apache License, Version 2.0
6			# (the "License"); you may not use this file except in compliance with
7			# the License. You may obtain a copy of the License at
8			#
9			# http://www.apache.org/licenses/LICENSE-2.0
10			#
11			# Unless required by applicable law or agreed to in writing, software
12			# distributed under the License is distributed on an "AS IS" BASIS,
13			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14			# See the License for the specific language governing permissions and
15			# limitations under the License.
16
17
18			"""
19
20			Tags: Ops tool.
21
22			A utility script that diffs models registered in st2 db versus what's on disk.
23
24			"""
25
26			import difflib
27			import json
28			import os
29			import sys
30
31			import eventlet
32			from oslo_config import cfg
33
34			from st2common import config
35			from st2common.constants.pack import DEFAULT_PACK_NAME
36			from st2common.content.loader import ContentPackLoader
37			from st2common.content.loader import MetaLoader
38			from st2common.bootstrap.base import ResourceRegistrar
39			import st2common.content.utils as content_utils
40			from st2common.models.api.action import ActionAPI
41			from st2common.models.api.sensor import SensorTypeAPI
42			from st2common.models.api.rule import RuleAPI
43			from st2common.models.db import db_setup
44			from st2common.models.db import db_teardown
45			from st2common.models.system.common import ResourceReference
46			from st2common.persistence.rule import Rule
47			from st2common.persistence.sensor import SensorType
48			from st2common.persistence.action import Action
49
50			registrar = ResourceRegistrar()
51			registrar.ALLOWED_EXTENSIONS = ['.yaml', '.yml', '.json']
52
53			meta_loader = MetaLoader()
54
55			API_MODELS_ARTIFACT_TYPES = {
56			'actions': ActionAPI,
57			'sensors': SensorTypeAPI,
58			'rules': RuleAPI
59			}
60
61			API_MODELS_PERSISTENT_MODELS = {
62			Action: ActionAPI,
63			SensorType: SensorTypeAPI,
64			Rule: RuleAPI
65			}
66
67
68			def do_register_cli_opts(opts, ignore_errors=False):
69			for opt in opts:
70			try:
71			cfg.CONF.register_cli_opt(opt)
72			except:
73			if not ignore_errors:
74			raise
75
76
77			def _monkey_patch():
78			eventlet.monkey_patch(
79			os=True,
80			select=True,
81			socket=True,
82			thread=False if '--use-debugger' in sys.argv else True,
83			time=True)
84
85
86			def _get_api_models_from_db(persistence_model, pack_dir=None):
87			filters = {}
88			if pack_dir:
89			pack_name = os.path.basename(os.path.normpath(pack_dir))
90			filters = {'pack': pack_name}
91			models = persistence_model.query(**filters)
92			models_dict = {}
93			for model in models:
94			model_pack = getattr(model, 'pack', None) or DEFAULT_PACK_NAME
95			model_ref = ResourceReference.to_string_reference(name=model.name, pack=model_pack)
96			if getattr(model, 'id', None):
97			del model.id
98			API_MODEL = API_MODELS_PERSISTENT_MODELS[persistence_model]
99			models_dict[model_ref] = API_MODEL.from_model(model)
100			return models_dict
101
102
103			def _get_api_models_from_disk(artifact_type, pack_dir=None):
104			loader = ContentPackLoader()
105			artifacts = None
106
107			if pack_dir:
108			artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
109			pack_name = os.path.basename(os.path.normpath(pack_dir))
110			artifacts = {pack_name: artifacts_dir}
111			else:
112			packs_dirs = content_utils.get_packs_base_paths()
113			artifacts = loader.get_content(packs_dirs, artifact_type)
114
115			artifacts_dict = {}
116			for pack_name, pack_path in artifacts.items():
117			artifacts_paths = registrar.get_resources_from_pack(pack_path)
118			for artifact_path in artifacts_paths:
119			artifact = meta_loader.load(artifact_path)
120			if artifact_type == 'sensors':
121			sensors_dir = os.path.dirname(artifact_path)
122			sensor_file_path = os.path.join(sensors_dir, artifact['entry_point'])
123			artifact['artifact_uri'] = 'file://' + sensor_file_path
124			name = artifact.get('name', None) or artifact.get('class_name', None)
125			if not artifact.get('pack', None):
126			artifact['pack'] = pack_name
127			ref = ResourceReference.to_string_reference(name=name,
128			pack=pack_name)
129			API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
130			# Following conversions are required because we add some fields with
131			# default values in db model. If we don't do these conversions,
132			# we'll see a unnecessary diff for those fields.
133			artifact_api = API_MODEL(**artifact)
134			artifact_db = API_MODEL.to_model(artifact_api)
135			artifact_api = API_MODEL.from_model(artifact_db)
136			artifacts_dict[ref] = artifact_api
137
138			return artifacts_dict
139
140
141			def _content_diff(artifact_type=None, artifact_in_disk=None, artifact_in_db=None,
142			verbose=False):
143			artifact_in_disk_str = json.dumps(
144			artifact_in_disk.__json__(), sort_keys=True,
145			indent=4, separators=(',', ': ')
146			)
147			artifact_in_db_str = json.dumps(
148			artifact_in_db.__json__(), sort_keys=True,
149			indent=4, separators=(',', ': ')
150			)
151			diffs = difflib.context_diff(artifact_in_db_str.splitlines(),
152			artifact_in_disk_str.splitlines(),
153			fromfile='DB contents', tofile='Disk contents')
154			printed = False
155			for diff in diffs:
156			if not printed:
157			identifier = getattr(artifact_in_db, 'ref', getattr(artifact_in_db, 'name'))
158			print('%s %s in db differs from what is in disk.' % (artifact_type.upper(),
159			identifier))
160			printed = True
161			print(diff)
162
163			if verbose:
164			print('\n\nOriginal contents:')
165			print('===================\n')
166			print('Artifact in db:\n\n%s\n\n' % artifact_in_db_str)
167			print('Artifact in disk:\n\n%s\n\n' % artifact_in_disk_str)
168
169
170			def _diff(persistence_model, artifact_type, pack_dir=None, verbose=True,
171			content_diff=True):
172			artifacts_in_db_dict = _get_api_models_from_db(persistence_model, pack_dir=pack_dir)
173			artifacts_in_disk_dict = _get_api_models_from_disk(artifact_type, pack_dir=pack_dir)
174
175			# print(artifacts_in_disk_dict)
176			all_artifacts = set(artifacts_in_db_dict.keys() + artifacts_in_disk_dict.keys())
177
178			for artifact in all_artifacts:
179			artifact_in_db = artifacts_in_db_dict.get(artifact, None)
180			artifact_in_disk = artifacts_in_disk_dict.get(artifact, None)
181			artifact_in_disk_pretty_json = None
182			artifact_in_db_pretty_json = None
183
184			if verbose:
185			print('******************************************************************************')
186			print('Checking if artifact %s is present in both disk and db.' % artifact)
187			if not artifact_in_db:
188			print('##############################################################################')
189			print('%s %s in disk not available in db.' % (artifact_type.upper(), artifact))
190			artifact_in_disk_pretty_json = json.dumps(
191			artifact_in_disk.__json__(), sort_keys=True,
192			indent=4, separators=(',', ': ')
193			)
194			if verbose:
195			print('File contents: \n')
196			print(artifact_in_disk_pretty_json)
197			continue
198
199			if not artifact_in_disk:
200			print('##############################################################################')
201			print('%s %s in db not available in disk.' % (artifact_type.upper(), artifact))
202			artifact_in_db_pretty_json = json.dumps(
203			artifact_in_db.__json__(), sort_keys=True,
204			indent=4, separators=(',', ': ')
205			)
206			if verbose:
207			print('DB contents: \n')
208			print(artifact_in_db_pretty_json)
209			continue
210			if verbose:
211			print('Artifact %s exists in both disk and db.' % artifact)
212
213			if content_diff:
214			if verbose:
215			print('Performing content diff for artifact %s.' % artifact)
216
217			_content_diff(artifact_type=artifact_type,
218			artifact_in_disk=artifact_in_disk,
219			artifact_in_db=artifact_in_db,
220			verbose=verbose)
221
222
223			def _diff_actions(pack_dir=None, verbose=False, content_diff=True):
224			_diff(Action, 'actions', pack_dir=pack_dir,
225			verbose=verbose, content_diff=content_diff)
226
227
228			def _diff_sensors(pack_dir=None, verbose=False, content_diff=True):
229			_diff(SensorType, 'sensors', pack_dir=pack_dir,
230			verbose=verbose, content_diff=content_diff)
231
232
233			def _diff_rules(pack_dir=None, verbose=True, content_diff=True):
234			_diff(Rule, 'rules', pack_dir=pack_dir,
235			verbose=verbose, content_diff=content_diff)
236
237
238			def main():
239			_monkey_patch()
240
241			cli_opts = [
242			cfg.BoolOpt('sensors', default=False,
243			help='diff sensor alone.'),
244			cfg.BoolOpt('actions', default=False,
245			help='diff actions alone.'),
246			cfg.BoolOpt('rules', default=False,
247			help='diff rules alone.'),
248			cfg.BoolOpt('all', default=False,
249			help='diff sensors, actions and rules.'),
250			cfg.BoolOpt('verbose', default=False),
251			cfg.BoolOpt('simple', default=False,
252			help='In simple mode, tool only tells you if content is missing.' +
253			'It doesn\'t show you content diff between disk and db.'),
254			cfg.StrOpt('pack-dir', default=None, help='Path to specific pack to diff.')
255			]
256			do_register_cli_opts(cli_opts)
257			config.parse_args()
258
259			username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
260			password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
261
262			# Connect to db.
263			db_setup(cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
264			username=username, password=password)
265
266			# Diff content
267			pack_dir = cfg.CONF.pack_dir or None
268			content_diff = not cfg.CONF.simple
269
270			if cfg.CONF.all:
271			_diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
272			_diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
273			_diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
274			return
275
276			if cfg.CONF.sensors:
277			_diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
278
279			if cfg.CONF.actions:
280			_diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
281
282			if cfg.CONF.rules:
283			_diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
284
285			# Disconnect from db.
286			db_teardown()
287
288			if __name__ == '__main__':
289			main()
290

StackStorm / st2

Pull Request — master (#2304)

print_load_avg() F

Complexity

Size

Duplication

How to fix Complexity

Complexity

Duplication Side-by-Side

Filter issues like