main() - Code Metrics - Inspection of "Remove Bintray deployments" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Pull Request — master (#2539)

by Arma

created 2016-02-29 15:08 UTC

main() C

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Metric	Value
cc	7
dl	0
loc	49
rs	5.5

#!/usr/bin/env python
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""

Tags: Ops tool.

A utility script that diffs models registered in st2 db versus what's on disk.

"""

import difflib
import json
import os

from oslo_config import cfg

from st2common import config
from st2common.util.monkey_patch import monkey_patch
from st2common.constants.pack import DEFAULT_PACK_NAME
from st2common.content.loader import ContentPackLoader
from st2common.content.loader import MetaLoader
from st2common.bootstrap.base import ResourceRegistrar
import st2common.content.utils as content_utils
from st2common.models.api.action import ActionAPI
from st2common.models.api.sensor import SensorTypeAPI
from st2common.models.api.rule import RuleAPI
from st2common.models.db import db_setup
from st2common.models.db import db_teardown
from st2common.models.system.common import ResourceReference
from st2common.persistence.rule import Rule
from st2common.persistence.sensor import SensorType
from st2common.persistence.action import Action

registrar = ResourceRegistrar()
registrar.ALLOWED_EXTENSIONS = ['.yaml', '.yml', '.json']

meta_loader = MetaLoader()

API_MODELS_ARTIFACT_TYPES = {
    'actions': ActionAPI,
    'sensors': SensorTypeAPI,
    'rules': RuleAPI
}

API_MODELS_PERSISTENT_MODELS = {
    Action: ActionAPI,
    SensorType: SensorTypeAPI,
    Rule: RuleAPI
}


def do_register_cli_opts(opts, ignore_errors=False):
    for opt in opts:
        try:
            cfg.CONF.register_cli_opt(opt)
        except:
            if not ignore_errors:
                raise


def _get_api_models_from_db(persistence_model, pack_dir=None):
    filters = {}
    if pack_dir:
        pack_name = os.path.basename(os.path.normpath(pack_dir))
        filters = {'pack': pack_name}
    models = persistence_model.query(**filters)
    models_dict = {}
    for model in models:
        model_pack = getattr(model, 'pack', None) or DEFAULT_PACK_NAME
        model_ref = ResourceReference.to_string_reference(name=model.name, pack=model_pack)
        if getattr(model, 'id', None):
            del model.id
        API_MODEL = API_MODELS_PERSISTENT_MODELS[persistence_model]
        models_dict[model_ref] = API_MODEL.from_model(model)
    return models_dict


def _get_api_models_from_disk(artifact_type, pack_dir=None):
    loader = ContentPackLoader()
    artifacts = None

    if pack_dir:
        artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
        pack_name = os.path.basename(os.path.normpath(pack_dir))
        artifacts = {pack_name: artifacts_dir}
    else:
        packs_dirs = content_utils.get_packs_base_paths()
        artifacts = loader.get_content(packs_dirs, artifact_type)

    artifacts_dict = {}
    for pack_name, pack_path in artifacts.items():
        artifacts_paths = registrar.get_resources_from_pack(pack_path)
        for artifact_path in artifacts_paths:
            artifact = meta_loader.load(artifact_path)
            if artifact_type == 'sensors':
                sensors_dir = os.path.dirname(artifact_path)
                sensor_file_path = os.path.join(sensors_dir, artifact['entry_point'])
                artifact['artifact_uri'] = 'file://' + sensor_file_path
            name = artifact.get('name', None) or artifact.get('class_name', None)
            if not artifact.get('pack', None):
                artifact['pack'] = pack_name
            ref = ResourceReference.to_string_reference(name=name,
                                                        pack=pack_name)
            API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
            # Following conversions are required because we add some fields with
            # default values in db model. If we don't do these conversions,
            # we'll see a unnecessary diff for those fields.
            artifact_api = API_MODEL(**artifact)
            artifact_db = API_MODEL.to_model(artifact_api)
            artifact_api = API_MODEL.from_model(artifact_db)
            artifacts_dict[ref] = artifact_api

    return artifacts_dict


def _content_diff(artifact_type=None, artifact_in_disk=None, artifact_in_db=None,
                  verbose=False):
    artifact_in_disk_str = json.dumps(
        artifact_in_disk.__json__(), sort_keys=True,
        indent=4, separators=(',', ': ')
    )
    artifact_in_db_str = json.dumps(
        artifact_in_db.__json__(), sort_keys=True,
        indent=4, separators=(',', ': ')
    )
    diffs = difflib.context_diff(artifact_in_db_str.splitlines(),
                                 artifact_in_disk_str.splitlines(),
                                 fromfile='DB contents', tofile='Disk contents')
    printed = False
    for diff in diffs:
        if not printed:
            identifier = getattr(artifact_in_db, 'ref', getattr(artifact_in_db, 'name'))
            print('%s %s in db differs from what is in disk.' % (artifact_type.upper(),
                  identifier))
            printed = True
        print(diff)

    if verbose:
        print('\n\nOriginal contents:')
        print('===================\n')
        print('Artifact in db:\n\n%s\n\n' % artifact_in_db_str)
        print('Artifact in disk:\n\n%s\n\n' % artifact_in_disk_str)


def _diff(persistence_model, artifact_type, pack_dir=None, verbose=True,
          content_diff=True):
    artifacts_in_db_dict = _get_api_models_from_db(persistence_model, pack_dir=pack_dir)
    artifacts_in_disk_dict = _get_api_models_from_disk(artifact_type, pack_dir=pack_dir)

    # print(artifacts_in_disk_dict)
    all_artifacts = set(artifacts_in_db_dict.keys() + artifacts_in_disk_dict.keys())

    for artifact in all_artifacts:
        artifact_in_db = artifacts_in_db_dict.get(artifact, None)
        artifact_in_disk = artifacts_in_disk_dict.get(artifact, None)
        artifact_in_disk_pretty_json = None
        artifact_in_db_pretty_json = None

        if verbose:
            print('******************************************************************************')
            print('Checking if artifact %s is present in both disk and db.' % artifact)
        if not artifact_in_db:
            print('##############################################################################')
            print('%s %s in disk not available in db.' % (artifact_type.upper(), artifact))
            artifact_in_disk_pretty_json = json.dumps(
                artifact_in_disk.__json__(), sort_keys=True,
                indent=4, separators=(',', ': ')
            )
            if verbose:
                print('File contents: \n')
                print(artifact_in_disk_pretty_json)
            continue

        if not artifact_in_disk:
            print('##############################################################################')
            print('%s %s in db not available in disk.' % (artifact_type.upper(), artifact))
            artifact_in_db_pretty_json = json.dumps(
                artifact_in_db.__json__(), sort_keys=True,
                indent=4, separators=(',', ': ')
            )
            if verbose:
                print('DB contents: \n')
                print(artifact_in_db_pretty_json)
            continue
        if verbose:
            print('Artifact %s exists in both disk and db.' % artifact)

        if content_diff:
            if verbose:
                print('Performing content diff for artifact %s.' % artifact)

            _content_diff(artifact_type=artifact_type,
                          artifact_in_disk=artifact_in_disk,
                          artifact_in_db=artifact_in_db,
                          verbose=verbose)


def _diff_actions(pack_dir=None, verbose=False, content_diff=True):
    _diff(Action, 'actions', pack_dir=pack_dir,
          verbose=verbose, content_diff=content_diff)


def _diff_sensors(pack_dir=None, verbose=False, content_diff=True):
    _diff(SensorType, 'sensors', pack_dir=pack_dir,
          verbose=verbose, content_diff=content_diff)


def _diff_rules(pack_dir=None, verbose=True, content_diff=True):
    _diff(Rule, 'rules', pack_dir=pack_dir,
          verbose=verbose, content_diff=content_diff)


def main():
    monkey_patch()

    cli_opts = [
        cfg.BoolOpt('sensors', default=False,
                    help='diff sensor alone.'),
        cfg.BoolOpt('actions', default=False,
                    help='diff actions alone.'),
        cfg.BoolOpt('rules', default=False,
                    help='diff rules alone.'),
        cfg.BoolOpt('all', default=False,
                    help='diff sensors, actions and rules.'),
        cfg.BoolOpt('verbose', default=False),
        cfg.BoolOpt('simple', default=False,
                    help='In simple mode, tool only tells you if content is missing.' +
                         'It doesn\'t show you content diff between disk and db.'),
        cfg.StrOpt('pack-dir', default=None, help='Path to specific pack to diff.')
    ]
    do_register_cli_opts(cli_opts)
    config.parse_args()

    username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
    password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None

    # Connect to db.
    db_setup(cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
             username=username, password=password)

    # Diff content
    pack_dir = cfg.CONF.pack_dir or None
    content_diff = not cfg.CONF.simple

    if cfg.CONF.all:
        _diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
        _diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
        _diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
        return

    if cfg.CONF.sensors:
        _diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)

    if cfg.CONF.actions:
        _diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)

    if cfg.CONF.rules:
        _diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)

    # Disconnect from db.
    db_teardown()

if __name__ == '__main__':
    main()


1			#!/usr/bin/env python
2			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
3			# contributor license agreements. See the NOTICE file distributed with
4			# this work for additional information regarding copyright ownership.
5			# The ASF licenses this file to You under the Apache License, Version 2.0
6			# (the "License"); you may not use this file except in compliance with
7			# the License. You may obtain a copy of the License at
8			#
9			# http://www.apache.org/licenses/LICENSE-2.0
10			#
11			# Unless required by applicable law or agreed to in writing, software
12			# distributed under the License is distributed on an "AS IS" BASIS,
13			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14			# See the License for the specific language governing permissions and
15			# limitations under the License.
16
17			"""
18
19			Tags: Ops tool.
20
21			A utility script that diffs models registered in st2 db versus what's on disk.
22
23			"""
24
25			import difflib
26			import json
27			import os
28
29			from oslo_config import cfg
30
31			from st2common import config
32			from st2common.util.monkey_patch import monkey_patch
33			from st2common.constants.pack import DEFAULT_PACK_NAME
34			from st2common.content.loader import ContentPackLoader
35			from st2common.content.loader import MetaLoader
36			from st2common.bootstrap.base import ResourceRegistrar
37			import st2common.content.utils as content_utils
38			from st2common.models.api.action import ActionAPI
39			from st2common.models.api.sensor import SensorTypeAPI
40			from st2common.models.api.rule import RuleAPI
41			from st2common.models.db import db_setup
42			from st2common.models.db import db_teardown
43			from st2common.models.system.common import ResourceReference
44			from st2common.persistence.rule import Rule
45			from st2common.persistence.sensor import SensorType
46			from st2common.persistence.action import Action
47
48			registrar = ResourceRegistrar()
49			registrar.ALLOWED_EXTENSIONS = ['.yaml', '.yml', '.json']
50
51			meta_loader = MetaLoader()
52
53			API_MODELS_ARTIFACT_TYPES = {
54			'actions': ActionAPI,
55			'sensors': SensorTypeAPI,
56			'rules': RuleAPI
57			}
58
59			API_MODELS_PERSISTENT_MODELS = {
60			Action: ActionAPI,
61			SensorType: SensorTypeAPI,
62			Rule: RuleAPI
63			}
64
65
66			def do_register_cli_opts(opts, ignore_errors=False):
67			for opt in opts:
68			try:
69			cfg.CONF.register_cli_opt(opt)
70			except:
71			if not ignore_errors:
72			raise
73
74
75			def _get_api_models_from_db(persistence_model, pack_dir=None):
76			filters = {}
77			if pack_dir:
78			pack_name = os.path.basename(os.path.normpath(pack_dir))
79			filters = {'pack': pack_name}
80			models = persistence_model.query(**filters)
81			models_dict = {}
82			for model in models:
83			model_pack = getattr(model, 'pack', None) or DEFAULT_PACK_NAME
84			model_ref = ResourceReference.to_string_reference(name=model.name, pack=model_pack)
85			if getattr(model, 'id', None):
86			del model.id
87			API_MODEL = API_MODELS_PERSISTENT_MODELS[persistence_model]
88			models_dict[model_ref] = API_MODEL.from_model(model)
89			return models_dict
90
91
92			def _get_api_models_from_disk(artifact_type, pack_dir=None):
93			loader = ContentPackLoader()
94			artifacts = None
95
96			if pack_dir:
97			artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
98			pack_name = os.path.basename(os.path.normpath(pack_dir))
99			artifacts = {pack_name: artifacts_dir}
100			else:
101			packs_dirs = content_utils.get_packs_base_paths()
102			artifacts = loader.get_content(packs_dirs, artifact_type)
103
104			artifacts_dict = {}
105			for pack_name, pack_path in artifacts.items():
106			artifacts_paths = registrar.get_resources_from_pack(pack_path)
107			for artifact_path in artifacts_paths:
108			artifact = meta_loader.load(artifact_path)
109			if artifact_type == 'sensors':
110			sensors_dir = os.path.dirname(artifact_path)
111			sensor_file_path = os.path.join(sensors_dir, artifact['entry_point'])
112			artifact['artifact_uri'] = 'file://' + sensor_file_path
113			name = artifact.get('name', None) or artifact.get('class_name', None)
114			if not artifact.get('pack', None):
115			artifact['pack'] = pack_name
116			ref = ResourceReference.to_string_reference(name=name,
117			pack=pack_name)
118			API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
119			# Following conversions are required because we add some fields with
120			# default values in db model. If we don't do these conversions,
121			# we'll see a unnecessary diff for those fields.
122			artifact_api = API_MODEL(**artifact)
123			artifact_db = API_MODEL.to_model(artifact_api)
124			artifact_api = API_MODEL.from_model(artifact_db)
125			artifacts_dict[ref] = artifact_api
126
127			return artifacts_dict
128
129
130			def _content_diff(artifact_type=None, artifact_in_disk=None, artifact_in_db=None,
131			verbose=False):
132			artifact_in_disk_str = json.dumps(
133			artifact_in_disk.__json__(), sort_keys=True,
134			indent=4, separators=(',', ': ')
135			)
136			artifact_in_db_str = json.dumps(
137			artifact_in_db.__json__(), sort_keys=True,
138			indent=4, separators=(',', ': ')
139			)
140			diffs = difflib.context_diff(artifact_in_db_str.splitlines(),
141			artifact_in_disk_str.splitlines(),
142			fromfile='DB contents', tofile='Disk contents')
143			printed = False
144			for diff in diffs:
145			if not printed:
146			identifier = getattr(artifact_in_db, 'ref', getattr(artifact_in_db, 'name'))
147			print('%s %s in db differs from what is in disk.' % (artifact_type.upper(),
148			identifier))
149			printed = True
150			print(diff)
151
152			if verbose:
153			print('\n\nOriginal contents:')
154			print('===================\n')
155			print('Artifact in db:\n\n%s\n\n' % artifact_in_db_str)
156			print('Artifact in disk:\n\n%s\n\n' % artifact_in_disk_str)
157
158
159			def _diff(persistence_model, artifact_type, pack_dir=None, verbose=True,
160			content_diff=True):
161			artifacts_in_db_dict = _get_api_models_from_db(persistence_model, pack_dir=pack_dir)
162			artifacts_in_disk_dict = _get_api_models_from_disk(artifact_type, pack_dir=pack_dir)
163
164			# print(artifacts_in_disk_dict)
165			all_artifacts = set(artifacts_in_db_dict.keys() + artifacts_in_disk_dict.keys())
166
167			for artifact in all_artifacts:
168			artifact_in_db = artifacts_in_db_dict.get(artifact, None)
169			artifact_in_disk = artifacts_in_disk_dict.get(artifact, None)
170			artifact_in_disk_pretty_json = None
171			artifact_in_db_pretty_json = None
172
173			if verbose:
174			print('******************************************************************************')
175			print('Checking if artifact %s is present in both disk and db.' % artifact)
176			if not artifact_in_db:
177			print('##############################################################################')
178			print('%s %s in disk not available in db.' % (artifact_type.upper(), artifact))
179			artifact_in_disk_pretty_json = json.dumps(
180			artifact_in_disk.__json__(), sort_keys=True,
181			indent=4, separators=(',', ': ')
182			)
183			if verbose:
184			print('File contents: \n')
185			print(artifact_in_disk_pretty_json)
186			continue
187
188			if not artifact_in_disk:
189			print('##############################################################################')
190			print('%s %s in db not available in disk.' % (artifact_type.upper(), artifact))
191			artifact_in_db_pretty_json = json.dumps(
192			artifact_in_db.__json__(), sort_keys=True,
193			indent=4, separators=(',', ': ')
194			)
195			if verbose:
196			print('DB contents: \n')
197			print(artifact_in_db_pretty_json)
198			continue
199			if verbose:
200			print('Artifact %s exists in both disk and db.' % artifact)
201
202			if content_diff:
203			if verbose:
204			print('Performing content diff for artifact %s.' % artifact)
205
206			_content_diff(artifact_type=artifact_type,
207			artifact_in_disk=artifact_in_disk,
208			artifact_in_db=artifact_in_db,
209			verbose=verbose)
210
211
212			def _diff_actions(pack_dir=None, verbose=False, content_diff=True):
213			_diff(Action, 'actions', pack_dir=pack_dir,
214			verbose=verbose, content_diff=content_diff)
215
216
217			def _diff_sensors(pack_dir=None, verbose=False, content_diff=True):
218			_diff(SensorType, 'sensors', pack_dir=pack_dir,
219			verbose=verbose, content_diff=content_diff)
220
221
222			def _diff_rules(pack_dir=None, verbose=True, content_diff=True):
223			_diff(Rule, 'rules', pack_dir=pack_dir,
224			verbose=verbose, content_diff=content_diff)
225
226
227			def main():
228			monkey_patch()
229
230			cli_opts = [
231			cfg.BoolOpt('sensors', default=False,
232			help='diff sensor alone.'),
233			cfg.BoolOpt('actions', default=False,
234			help='diff actions alone.'),
235			cfg.BoolOpt('rules', default=False,
236			help='diff rules alone.'),
237			cfg.BoolOpt('all', default=False,
238			help='diff sensors, actions and rules.'),
239			cfg.BoolOpt('verbose', default=False),
240			cfg.BoolOpt('simple', default=False,
241			help='In simple mode, tool only tells you if content is missing.' +
242			'It doesn\'t show you content diff between disk and db.'),
243			cfg.StrOpt('pack-dir', default=None, help='Path to specific pack to diff.')
244			]
245			do_register_cli_opts(cli_opts)
246			config.parse_args()
247
248			username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
249			password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
250
251			# Connect to db.
252			db_setup(cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
253			username=username, password=password)
254
255			# Diff content
256			pack_dir = cfg.CONF.pack_dir or None
257			content_diff = not cfg.CONF.simple
258
259			if cfg.CONF.all:
260			_diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
261			_diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
262			_diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
263			return
264
265			if cfg.CONF.sensors:
266			_diff_sensors(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
267
268			if cfg.CONF.actions:
269			_diff_actions(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
270
271			if cfg.CONF.rules:
272			_diff_rules(pack_dir=pack_dir, verbose=cfg.CONF.verbose, content_diff=content_diff)
273
274			# Disconnect from db.
275			db_teardown()
276
277			if __name__ == '__main__':
278			main()
279

StackStorm / st2

Pull Request — master (#2539)

main() C

Complexity

Size

Duplication

Duplication Side-by-Side

Filter issues like