st2debug.cmd.create_archive() - Code Metrics - Inspection of "Merge pull request #2246 from StackStorm/chatops/c..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( d1f0a7...3b2ece )

by Edward

created 2015-11-30 10:28 UTC

st2debug.cmd.create_archive() F

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

111

Duplication

Lines	0
Ratio	0 %

Metric	Value
dl	0
loc	111
rs	2
cc	18

How to fix Long Method Complexity

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This script submits information which helps StackStorm employees debug different
user problems and issues to StackStorm.

By default the following information is included:

- Logs from /var/log/st2
- StackStorm and mistral config file (/etc/st2/st2.conf, /etc/mistral/mistral.conf)
- All the content (integration packs).
- Information about your system and StackStorm installation (Operating system,
  Python version, StackStorm version, Mistral version)

Note: This script currently assumes it's running on Linux.
"""

import os
import sys
import shutil
import socket
import logging
import tarfile
import argparse
import platform
import tempfile
import httplib

import six
import yaml
import gnupg
import requests
from distutils.spawn import find_executable

import st2common
from st2common.content.utils import get_packs_base_paths
from st2common import __version__ as st2_version
from st2common import config
from st2common.util import date as date_utils
from st2debug.constants import GPG_KEY
from st2debug.constants import GPG_KEY_FINGERPRINT
from st2debug.constants import S3_BUCKET_URL
from st2debug.utils.fs import copy_files
from st2debug.utils.fs import get_full_file_list
from st2debug.utils.fs import get_dirs_in_path
from st2debug.utils.fs import remove_file
from st2debug.utils.system_info import get_cpu_info
from st2debug.utils.system_info import get_memory_info
from st2debug.utils.system_info import get_package_list
from st2debug.utils.git_utils import get_repo_latest_revision_hash
from st2debug.processors import process_st2_config
from st2debug.processors import process_mistral_config
from st2debug.processors import process_content_pack_dir

LOG = logging.getLogger(__name__)

# Constants
GPG_INSTALLED = find_executable('gpg') is not None

ST2_LOG_FILES_PATH = '/var/log/st2/*.log'
MISTRAL_LOG_FILES_PATH = '/var/log/mistral*.log'

LOG_FILE_PATHS = [
    ST2_LOG_FILES_PATH,
    MISTRAL_LOG_FILES_PATH
]

ST2_CONFIG_FILE_PATH = '/etc/st2/st2.conf'
MISTRAL_CONFIG_FILE_PATH = '/etc/mistral/mistral.conf'

ST2_CONFIG_FILE_NAME = os.path.split(ST2_CONFIG_FILE_PATH)[1]
MISTRAL_CONFIG_FILE_NAME = os.path.split(MISTRAL_CONFIG_FILE_PATH)[1]

CONFIG_FILE_PATHS = [
    ST2_CONFIG_FILE_PATH,
    MISTRAL_CONFIG_FILE_PATH
]

# Directory structure inside tarball
DIRECTORY_STRUCTURE = [
    'configs/',
    'logs/',
    'content/'
]

# Options which should be removed from the st2 config
ST2_CONF_OPTIONS_TO_REMOVE = {
    'database': ['username', 'password'],
    'messaging': ['url']
}

REMOVE_VALUE_NAME = '**removed**'

OUTPUT_FILENAME_TEMPLATE = 'st2-debug-output-%(hostname)s-%(date)s.tar.gz'

try:
    config.parse_args(args=[])
except Exception:
    pass


def setup_logging():
    root = LOG
    root.setLevel(logging.INFO)

    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s  %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    root.addHandler(ch)


def get_system_information():
    """
    Retrieve system information which is included in the report.

    :rtype: ``dict``
    """
    system_information = {
        'hostname': socket.gethostname(),
        'operating_system': {},
        'hardware': {
            'cpu': {},
            'memory': {}
        },
        'python': {},
        'stackstorm': {},
        'mistral': {}
    }

    # Operating system information
    system_information['operating_system']['system'] = platform.system()
    system_information['operating_system']['release'] = platform.release()
    system_information['operating_system']['operating_system'] = platform.platform()
    system_information['operating_system']['platform'] = platform.system()
    system_information['operating_system']['architecture'] = ' '.join(platform.architecture())

    if platform.system().lower() == 'linux':
        distribution = ' '.join(platform.linux_distribution())
        system_information['operating_system']['distribution'] = distribution

    system_information['python']['version'] = sys.version.split('\n')[0]

    # Hardware information
    cpu_info = get_cpu_info()

    if cpu_info:
        core_count = len(cpu_info)
        model = cpu_info[0]['model_name']
        system_information['hardware']['cpu'] = {
            'core_count': core_count,
            'model_name': model
        }
    else:
        # Unsupported platform
        system_information['hardware']['cpu'] = 'unsupported platform'

    memory_info = get_memory_info()

    if memory_info:
        total = memory_info['MemTotal'] / 1024
        free = memory_info['MemFree'] / 1024
        used = (total - free)
        system_information['hardware']['memory'] = {
            'total': total,
            'used': used,
            'free': free
        }
    else:
        # Unsupported platform
        system_information['hardware']['memory'] = 'unsupported platform'

    # StackStorm information
    system_information['stackstorm']['version'] = st2_version

    st2common_path = st2common.__file__
    st2common_path = os.path.dirname(st2common_path)

    if 'st2common/st2common' in st2common_path:
        # Assume we are running source install
        base_install_path = st2common_path.replace('/st2common/st2common', '')

        revision_hash = get_repo_latest_revision_hash(repo_path=base_install_path)

        system_information['stackstorm']['installation_method'] = 'source'
        system_information['stackstorm']['revision_hash'] = revision_hash
    else:
        package_list = get_package_list(name_startswith='st2')

        system_information['stackstorm']['installation_method'] = 'package'
        system_information['stackstorm']['packages'] = package_list

    # Mistral information
    repo_path = '/opt/openstack/mistral'
    revision_hash = get_repo_latest_revision_hash(repo_path=repo_path)
    system_information['mistral']['installation_method'] = 'source'
    system_information['mistral']['revision_hash'] = revision_hash

    return system_information


def create_archive(include_logs, include_configs, include_content, include_system_info,
                   user_info=None, debug=False):
    """
    Create an archive with debugging information.

    :return: Path to the generated archive.
    :rtype: ``str``
    """
    date = date_utils.get_datetime_utc_now().strftime('%Y-%m-%d-%H:%M:%S')
    values = {'hostname': socket.gethostname(), 'date': date}

    output_file_name = OUTPUT_FILENAME_TEMPLATE % values
    output_file_path = os.path.join('/tmp', output_file_name)

    # 1. Create temporary directory with the final directory structure where we will move files
    # which will be processed and included in the tarball
    temp_dir_path = tempfile.mkdtemp()

    output_paths = {
        'logs': os.path.join(temp_dir_path, 'logs/'),
        'configs': os.path.join(temp_dir_path, 'configs/'),
        'content': os.path.join(temp_dir_path, 'content/'),
        'system_info': os.path.join(temp_dir_path, 'system_info.yaml'),
        'user_info': os.path.join(temp_dir_path, 'user_info.yaml')
    }

    for directory_name in DIRECTORY_STRUCTURE:
        full_path = os.path.join(temp_dir_path, directory_name)
        os.mkdir(full_path)

    # 2. Moves all the files to the temporary directory
    LOG.info('Collecting files...')

    # Logs
    if include_logs:
        LOG.debug('Including log files')

        for file_path_glob in LOG_FILE_PATHS:
            log_file_list = get_full_file_list(file_path_glob=file_path_glob)
            copy_files(file_paths=log_file_list, destination=output_paths['logs'])

    # Config files
    if include_configs:
        LOG.debug('Including config files')
        copy_files(file_paths=CONFIG_FILE_PATHS, destination=output_paths['configs'])

    # Content
    if include_content:
        LOG.debug('Including content')

        packs_base_paths = get_packs_base_paths()
        for index, packs_base_path in enumerate(packs_base_paths, 1):
            dst = os.path.join(output_paths['content'], 'dir-%s' % (index))

            try:
                shutil.copytree(src=packs_base_path, dst=dst)
            except IOError:
                continue

    # System information
    if include_system_info:
        LOG.debug('Including system info')

        system_information = get_system_information()
        system_information = yaml.dump(system_information, default_flow_style=False)

        with open(output_paths['system_info'], 'w') as fp:
            fp.write(system_information)

    if user_info:
        LOG.debug('Including user info')
        user_info = yaml.dump(user_info, default_flow_style=False)

        with open(output_paths['user_info'], 'w') as fp:
            fp.write(user_info)

    # Configs
    st2_config_path = os.path.join(output_paths['configs'], ST2_CONFIG_FILE_NAME)
    process_st2_config(config_path=st2_config_path)

    mistral_config_path = os.path.join(output_paths['configs'], MISTRAL_CONFIG_FILE_NAME)
    process_mistral_config(config_path=mistral_config_path)

    # Content
    base_pack_dirs = get_dirs_in_path(file_path=output_paths['content'])

    for base_pack_dir in base_pack_dirs:
        pack_dirs = get_dirs_in_path(file_path=base_pack_dir)

        for pack_dir in pack_dirs:
            process_content_pack_dir(pack_dir=pack_dir)

    # 4. Create a tarball
    LOG.info('Creating tarball...')

    with tarfile.open(output_file_path, 'w:gz') as tar:
        for file_path in output_paths.values():
            file_path = os.path.normpath(file_path)
            source_dir = file_path

            if not os.path.exists(source_dir):
                continue

            if '.' in file_path:
                arcname = os.path.basename(file_path)
            else:
                arcname = os.path.split(file_path)[-1]

            tar.add(source_dir, arcname=arcname)

    return output_file_path


def encrypt_archive(archive_file_path, debug=False):
    """
    Encrypt archive with debugging information using our public key.

    :param archive_file_path: Path to the non-encrypted tarball file.
    :type archive_file_path: ``str``

    :return: Path to the encrypted archive.
    :rtype: ``str``
    """
    assert archive_file_path.endswith('.tar.gz')

    LOG.info('Encrypting tarball...')
    gpg = gnupg.GPG(verbose=debug)

    # Import our public key
    import_result = gpg.import_keys(GPG_KEY)
    # pylint: disable=no-member
    assert import_result.count == 1

    encrypted_archive_output_file_path = archive_file_path + '.asc'
    with open(archive_file_path, 'rb') as fp:
        gpg.encrypt_file(fp,
                         recipients=GPG_KEY_FINGERPRINT,
                         always_trust=True,
                         output=encrypted_archive_output_file_path)

    return encrypted_archive_output_file_path


def upload_archive(archive_file_path):
    assert archive_file_path.endswith('.asc')

    LOG.debug('Uploading tarball...')
    files = {'file': open(archive_file_path, 'rb')}
    file_name = os.path.split(archive_file_path)[1]
    url = S3_BUCKET_URL + file_name
    assert url.startswith('https://')

    response = requests.put(url=url, files=files)
    assert response.status_code == httplib.OK


def create_and_review_archive(include_logs, include_configs, include_content, include_system_info,
                              user_info=None, debug=False):
    try:
        plain_text_output_path = create_archive(include_logs=include_logs,
                                                include_configs=include_configs,
                                                include_content=include_content,
                                                include_system_info=include_system_info,
                                                user_info=user_info,
                                                debug=debug)
    except Exception:
        LOG.exception('Failed to generate tarball', exc_info=True)
    else:
        LOG.info('Debug tarball successfully generated and can be reviewed at: %s' %

                 (plain_text_output_path))


def create_and_upload_archive(include_logs, include_configs, include_content, include_system_info,
                              user_info=None, debug=False):
    try:
        plain_text_output_path = create_archive(include_logs=include_logs,
                                                include_configs=include_configs,
                                                include_content=include_content,
                                                include_system_info=include_system_info,
                                                user_info=user_info,
                                                debug=debug)
        encrypted_output_path = encrypt_archive(archive_file_path=plain_text_output_path)
        upload_archive(archive_file_path=encrypted_output_path)
    except Exception:
        LOG.exception('Failed to upload tarball to StackStorm', exc_info=True)
        plain_text_output_path = None
        encrypted_output_path = None
    else:
        tarball_name = os.path.basename(encrypted_output_path)
        LOG.info('Debug tarball successfully uploaded to StackStorm (name=%s)' % (tarball_name))

        LOG.info('When communicating with support, please let them know the tarball name - %s' %

                 (tarball_name))

    finally:
        # Remove tarballs
        if plain_text_output_path:
            assert plain_text_output_path.startswith('/tmp')
            remove_file(file_path=plain_text_output_path)
        if encrypted_output_path:
            assert encrypted_output_path.startswith('/tmp')
            remove_file(file_path=encrypted_output_path)


def main():
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--exclude-logs', action='store_true', default=False,
                        help='Don\'t include logs in the generated tarball')
    parser.add_argument('--exclude-configs', action='store_true', default=False,
                        help='Don\'t include configs in the generated tarball')
    parser.add_argument('--exclude-content', action='store_true', default=False,
                        help='Don\'t include content packs in the generated tarball')
    parser.add_argument('--exclude-system-info', action='store_true', default=False,
                        help='Don\'t include system information in the generated tarball')
    parser.add_argument('--yes', action='store_true', default=False,
                        help='Run in non-interactive mode and answer "yes" to all the questions')
    parser.add_argument('--review', action='store_true', default=False,
                        help='Generate the tarball, but don\'t encrypt and upload it')
    parser.add_argument('--debug', action='store_true', default=False,
                        help='Enable debug mode')
    args = parser.parse_args()

    arg_names = ['exclude_logs', 'exclude_configs', 'exclude_content',
                 'exclude_system_info']

    abort = True
    for arg_name in arg_names:
        value = getattr(args, arg_name, False)
        abort &= value

    if abort:
        print('Generated tarball would be empty. Aborting.')
        sys.exit(2)

    submited_content = [name.replace('exclude_', '') for name in arg_names if
                        not getattr(args, name, False)]
    submited_content = ', '.join(submited_content)

    if not args.yes and not args.review:
        # When not running in review mode, GPG needs to be installed and
        # available
        if not GPG_INSTALLED:
            msg = ('"gpg" binary not found, can\'t proceed. Make sure "gpg" is installed '
                   'and available in PATH.')
            raise ValueError(msg)

        print('This will submit the following information to StackStorm: %s' % (submited_content))
        value = six.moves.input('Are you sure you want to proceed? [y/n] ')
        if value.strip().lower() not in ['y', 'yes']:
            print('Aborting')
            sys.exit(1)

    # Prompt user for optional additional context info
    user_info = {}
    if not args.yes:
        print('If you want us to get back to you via email, you can provide additional context '
              'such as your name, email and an optional comment')
        value = six.moves.input('Would you like to provide additional context? [y/n] ')
        if value.strip().lower() in ['y', 'yes']:
            user_info['name'] = six.moves.input('Name: ')
            user_info['email'] = six.moves.input('Email: ')
            user_info['comment'] = six.moves.input('Comment: ')

    setup_logging()

    if args.review:
        create_and_review_archive(include_logs=not args.exclude_logs,
                                  include_configs=not args.exclude_configs,
                                  include_content=not args.exclude_content,
                                  include_system_info=not args.exclude_system_info,
                                  user_info=user_info,
                                  debug=args.debug)
    else:
        create_and_upload_archive(include_logs=not args.exclude_logs,
                                  include_configs=not args.exclude_configs,
                                  include_content=not args.exclude_content,
                                  include_system_info=not args.exclude_system_info,
                                  user_info=user_info,
                                  debug=args.debug)


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			"""
17			This script submits information which helps StackStorm employees debug different
18			user problems and issues to StackStorm.
19
20			By default the following information is included:
21
22			- Logs from /var/log/st2
23			- StackStorm and mistral config file (/etc/st2/st2.conf, /etc/mistral/mistral.conf)
24			- All the content (integration packs).
25			- Information about your system and StackStorm installation (Operating system,
26			Python version, StackStorm version, Mistral version)
27
28			Note: This script currently assumes it's running on Linux.
29			"""
30
31			import os
32			import sys
33			import shutil
34			import socket
35			import logging
36			import tarfile
37			import argparse
38			import platform
39			import tempfile
40			import httplib
41
42			import six
43			import yaml
44			import gnupg
45			import requests
46			from distutils.spawn import find_executable
47
48			import st2common
49			from st2common.content.utils import get_packs_base_paths
50			from st2common import __version__ as st2_version
51			from st2common import config
52			from st2common.util import date as date_utils
53			from st2debug.constants import GPG_KEY
54			from st2debug.constants import GPG_KEY_FINGERPRINT
55			from st2debug.constants import S3_BUCKET_URL
56			from st2debug.utils.fs import copy_files
57			from st2debug.utils.fs import get_full_file_list
58			from st2debug.utils.fs import get_dirs_in_path
59			from st2debug.utils.fs import remove_file
60			from st2debug.utils.system_info import get_cpu_info
61			from st2debug.utils.system_info import get_memory_info
62			from st2debug.utils.system_info import get_package_list
63			from st2debug.utils.git_utils import get_repo_latest_revision_hash
64			from st2debug.processors import process_st2_config
65			from st2debug.processors import process_mistral_config
66			from st2debug.processors import process_content_pack_dir
67
68			LOG = logging.getLogger(__name__)
69
70			# Constants
71			GPG_INSTALLED = find_executable('gpg') is not None
72
73			ST2_LOG_FILES_PATH = '/var/log/st2/*.log'
74			MISTRAL_LOG_FILES_PATH = '/var/log/mistral*.log'
75
76			LOG_FILE_PATHS = [
77			ST2_LOG_FILES_PATH,
78			MISTRAL_LOG_FILES_PATH
79			]
80
81			ST2_CONFIG_FILE_PATH = '/etc/st2/st2.conf'
82			MISTRAL_CONFIG_FILE_PATH = '/etc/mistral/mistral.conf'
83
84			ST2_CONFIG_FILE_NAME = os.path.split(ST2_CONFIG_FILE_PATH)[1]
85			MISTRAL_CONFIG_FILE_NAME = os.path.split(MISTRAL_CONFIG_FILE_PATH)[1]
86
87			CONFIG_FILE_PATHS = [
88			ST2_CONFIG_FILE_PATH,
89			MISTRAL_CONFIG_FILE_PATH
90			]
91
92			# Directory structure inside tarball
93			DIRECTORY_STRUCTURE = [
94			'configs/',
95			'logs/',
96			'content/'
97			]
98
99			# Options which should be removed from the st2 config
100			ST2_CONF_OPTIONS_TO_REMOVE = {
101			'database': ['username', 'password'],
102			'messaging': ['url']
103			}
104
105			REMOVE_VALUE_NAME = 'removed'
106
107			OUTPUT_FILENAME_TEMPLATE = 'st2-debug-output-%(hostname)s-%(date)s.tar.gz'
108
109			try:
110			config.parse_args(args=[])
111			except Exception:
112			pass
113
114
115			def setup_logging():
116			root = LOG
117			root.setLevel(logging.INFO)
118
119			ch = logging.StreamHandler(sys.stdout)
120			ch.setLevel(logging.DEBUG)
121			formatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
122			ch.setFormatter(formatter)
123			root.addHandler(ch)
124
125
126			def get_system_information():
127			"""
128			Retrieve system information which is included in the report.
129
130			:rtype: ``dict``
131			"""
132			system_information = {
133			'hostname': socket.gethostname(),
134			'operating_system': {},
135			'hardware': {
136			'cpu': {},
137			'memory': {}
138			},
139			'python': {},
140			'stackstorm': {},
141			'mistral': {}
142			}
143
144			# Operating system information
145			system_information['operating_system']['system'] = platform.system()
146			system_information['operating_system']['release'] = platform.release()
147			system_information['operating_system']['operating_system'] = platform.platform()
148			system_information['operating_system']['platform'] = platform.system()
149			system_information['operating_system']['architecture'] = ' '.join(platform.architecture())
150
151			if platform.system().lower() == 'linux':
152			distribution = ' '.join(platform.linux_distribution())
153			system_information['operating_system']['distribution'] = distribution
154
155			system_information['python']['version'] = sys.version.split('\n')[0]
156
157			# Hardware information
158			cpu_info = get_cpu_info()
159
160			if cpu_info:
161			core_count = len(cpu_info)
162			model = cpu_info[0]['model_name']
163			system_information['hardware']['cpu'] = {
164			'core_count': core_count,
165			'model_name': model
166			}
167			else:
168			# Unsupported platform
169			system_information['hardware']['cpu'] = 'unsupported platform'
170
171			memory_info = get_memory_info()
172
173			if memory_info:
174			total = memory_info['MemTotal'] / 1024
175			free = memory_info['MemFree'] / 1024
176			used = (total - free)
177			system_information['hardware']['memory'] = {
178			'total': total,
179			'used': used,
180			'free': free
181			}
182			else:
183			# Unsupported platform
184			system_information['hardware']['memory'] = 'unsupported platform'
185
186			# StackStorm information
187			system_information['stackstorm']['version'] = st2_version
188
189			st2common_path = st2common.__file__
190			st2common_path = os.path.dirname(st2common_path)
191
192			if 'st2common/st2common' in st2common_path:
193			# Assume we are running source install
194			base_install_path = st2common_path.replace('/st2common/st2common', '')
195
196			revision_hash = get_repo_latest_revision_hash(repo_path=base_install_path)
197
198			system_information['stackstorm']['installation_method'] = 'source'
199			system_information['stackstorm']['revision_hash'] = revision_hash
200			else:
201			package_list = get_package_list(name_startswith='st2')
202
203			system_information['stackstorm']['installation_method'] = 'package'
204			system_information['stackstorm']['packages'] = package_list
205
206			# Mistral information
207			repo_path = '/opt/openstack/mistral'
208			revision_hash = get_repo_latest_revision_hash(repo_path=repo_path)
209			system_information['mistral']['installation_method'] = 'source'
210			system_information['mistral']['revision_hash'] = revision_hash
211
212			return system_information
213
214
215			def create_archive(include_logs, include_configs, include_content, include_system_info,
216			user_info=None, debug=False):
217			"""
218			Create an archive with debugging information.
219
220			:return: Path to the generated archive.
221			:rtype: ``str``
222			"""
223			date = date_utils.get_datetime_utc_now().strftime('%Y-%m-%d-%H:%M:%S')
224			values = {'hostname': socket.gethostname(), 'date': date}
225
226			output_file_name = OUTPUT_FILENAME_TEMPLATE % values
227			output_file_path = os.path.join('/tmp', output_file_name)
228
229			# 1. Create temporary directory with the final directory structure where we will move files
230			# which will be processed and included in the tarball
231			temp_dir_path = tempfile.mkdtemp()
232
233			output_paths = {
234			'logs': os.path.join(temp_dir_path, 'logs/'),
235			'configs': os.path.join(temp_dir_path, 'configs/'),
236			'content': os.path.join(temp_dir_path, 'content/'),
237			'system_info': os.path.join(temp_dir_path, 'system_info.yaml'),
238			'user_info': os.path.join(temp_dir_path, 'user_info.yaml')
239			}
240
241			for directory_name in DIRECTORY_STRUCTURE:
242			full_path = os.path.join(temp_dir_path, directory_name)
243			os.mkdir(full_path)
244
245			# 2. Moves all the files to the temporary directory
246			LOG.info('Collecting files...')
247
248			# Logs
249			if include_logs:
250			LOG.debug('Including log files')
251
252			for file_path_glob in LOG_FILE_PATHS:
253			log_file_list = get_full_file_list(file_path_glob=file_path_glob)
254			copy_files(file_paths=log_file_list, destination=output_paths['logs'])
255
256			# Config files
257			if include_configs:
258			LOG.debug('Including config files')
259			copy_files(file_paths=CONFIG_FILE_PATHS, destination=output_paths['configs'])
260
261			# Content
262			if include_content:
263			LOG.debug('Including content')
264
265			packs_base_paths = get_packs_base_paths()
266			for index, packs_base_path in enumerate(packs_base_paths, 1):
267			dst = os.path.join(output_paths['content'], 'dir-%s' % (index))
268
269			try:
270			shutil.copytree(src=packs_base_path, dst=dst)
271			except IOError:
272			continue
273
274			# System information
275			if include_system_info:
276			LOG.debug('Including system info')
277
278			system_information = get_system_information()
279			system_information = yaml.dump(system_information, default_flow_style=False)
280
281			with open(output_paths['system_info'], 'w') as fp:
282			fp.write(system_information)
283
284			if user_info:
285			LOG.debug('Including user info')
286			user_info = yaml.dump(user_info, default_flow_style=False)
287
288			with open(output_paths['user_info'], 'w') as fp:
289			fp.write(user_info)
290
291			# Configs
292			st2_config_path = os.path.join(output_paths['configs'], ST2_CONFIG_FILE_NAME)
293			process_st2_config(config_path=st2_config_path)
294
295			mistral_config_path = os.path.join(output_paths['configs'], MISTRAL_CONFIG_FILE_NAME)
296			process_mistral_config(config_path=mistral_config_path)
297
298			# Content
299			base_pack_dirs = get_dirs_in_path(file_path=output_paths['content'])
300
301			for base_pack_dir in base_pack_dirs:
302			pack_dirs = get_dirs_in_path(file_path=base_pack_dir)
303
304			for pack_dir in pack_dirs:
305			process_content_pack_dir(pack_dir=pack_dir)
306
307			# 4. Create a tarball
308			LOG.info('Creating tarball...')
309
310			with tarfile.open(output_file_path, 'w:gz') as tar:
311			for file_path in output_paths.values():
312			file_path = os.path.normpath(file_path)
313			source_dir = file_path
314
315			if not os.path.exists(source_dir):
316			continue
317
318			if '.' in file_path:
319			arcname = os.path.basename(file_path)
320			else:
321			arcname = os.path.split(file_path)[-1]
322
323			tar.add(source_dir, arcname=arcname)
324
325			return output_file_path
326
327
328			def encrypt_archive(archive_file_path, debug=False):
329			"""
330			Encrypt archive with debugging information using our public key.
331
332			:param archive_file_path: Path to the non-encrypted tarball file.
333			:type archive_file_path: ``str``
334
335			:return: Path to the encrypted archive.
336			:rtype: ``str``
337			"""
338			assert archive_file_path.endswith('.tar.gz')
339
340			LOG.info('Encrypting tarball...')
341			gpg = gnupg.GPG(verbose=debug)
342
343			# Import our public key
344			import_result = gpg.import_keys(GPG_KEY)
345			# pylint: disable=no-member
346			assert import_result.count == 1
347
348			encrypted_archive_output_file_path = archive_file_path + '.asc'
349			with open(archive_file_path, 'rb') as fp:
350			gpg.encrypt_file(fp,
351			recipients=GPG_KEY_FINGERPRINT,
352			always_trust=True,
353			output=encrypted_archive_output_file_path)
354
355			return encrypted_archive_output_file_path
356
357
358			def upload_archive(archive_file_path):
359			assert archive_file_path.endswith('.asc')
360
361			LOG.debug('Uploading tarball...')
362			files = {'file': open(archive_file_path, 'rb')}
363			file_name = os.path.split(archive_file_path)[1]
364			url = S3_BUCKET_URL + file_name
365			assert url.startswith('https://')
366
367			response = requests.put(url=url, files=files)
368			assert response.status_code == httplib.OK
369
370
371			def create_and_review_archive(include_logs, include_configs, include_content, include_system_info,
372			user_info=None, debug=False):
373			try:
374			plain_text_output_path = create_archive(include_logs=include_logs,
375			include_configs=include_configs,
376			include_content=include_content,
377			include_system_info=include_system_info,
378			user_info=user_info,
379			debug=debug)
380			except Exception:
381			LOG.exception('Failed to generate tarball', exc_info=True)
382			else:
383			LOG.info('Debug tarball successfully generated and can be reviewed at: %s' %
			0 ignored issues – show Coding Style Best Practice introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report Specify string format arguments as logging function parameters Loading history...
384			(plain_text_output_path))
385
386
387			def create_and_upload_archive(include_logs, include_configs, include_content, include_system_info,
388			user_info=None, debug=False):
389			try:
390			plain_text_output_path = create_archive(include_logs=include_logs,
391			include_configs=include_configs,
392			include_content=include_content,
393			include_system_info=include_system_info,
394			user_info=user_info,
395			debug=debug)
396			encrypted_output_path = encrypt_archive(archive_file_path=plain_text_output_path)
397			upload_archive(archive_file_path=encrypted_output_path)
398			except Exception:
399			LOG.exception('Failed to upload tarball to StackStorm', exc_info=True)
400			plain_text_output_path = None
401			encrypted_output_path = None
402			else:
403			tarball_name = os.path.basename(encrypted_output_path)
404			LOG.info('Debug tarball successfully uploaded to StackStorm (name=%s)' % (tarball_name))
			0 ignored issues – show Coding Style Best Practice introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report Specify string format arguments as logging function parameters Loading history...
405			LOG.info('When communicating with support, please let them know the tarball name - %s' %
			0 ignored issues – show Coding Style Best Practice introduced 2015-11-20 23:04 UTC by Report Bug Copy Issue Report Specify string format arguments as logging function parameters Loading history...
406			(tarball_name))
407
408			finally:
409			# Remove tarballs
410			if plain_text_output_path:
411			assert plain_text_output_path.startswith('/tmp')
412			remove_file(file_path=plain_text_output_path)
413			if encrypted_output_path:
414			assert encrypted_output_path.startswith('/tmp')
415			remove_file(file_path=encrypted_output_path)
416
417
418			def main():
419			parser = argparse.ArgumentParser(description='')
420			parser.add_argument('--exclude-logs', action='store_true', default=False,
421			help='Don\'t include logs in the generated tarball')
422			parser.add_argument('--exclude-configs', action='store_true', default=False,
423			help='Don\'t include configs in the generated tarball')
424			parser.add_argument('--exclude-content', action='store_true', default=False,
425			help='Don\'t include content packs in the generated tarball')
426			parser.add_argument('--exclude-system-info', action='store_true', default=False,
427			help='Don\'t include system information in the generated tarball')
428			parser.add_argument('--yes', action='store_true', default=False,
429			help='Run in non-interactive mode and answer "yes" to all the questions')
430			parser.add_argument('--review', action='store_true', default=False,
431			help='Generate the tarball, but don\'t encrypt and upload it')
432			parser.add_argument('--debug', action='store_true', default=False,
433			help='Enable debug mode')
434			args = parser.parse_args()
435
436			arg_names = ['exclude_logs', 'exclude_configs', 'exclude_content',
437			'exclude_system_info']
438
439			abort = True
440			for arg_name in arg_names:
441			value = getattr(args, arg_name, False)
442			abort &= value
443
444			if abort:
445			print('Generated tarball would be empty. Aborting.')
446			sys.exit(2)
447
448			submited_content = [name.replace('exclude_', '') for name in arg_names if
449			not getattr(args, name, False)]
450			submited_content = ', '.join(submited_content)
451
452			if not args.yes and not args.review:
453			# When not running in review mode, GPG needs to be installed and
454			# available
455			if not GPG_INSTALLED:
456			msg = ('"gpg" binary not found, can\'t proceed. Make sure "gpg" is installed '
457			'and available in PATH.')
458			raise ValueError(msg)
459
460			print('This will submit the following information to StackStorm: %s' % (submited_content))
461			value = six.moves.input('Are you sure you want to proceed? [y/n] ')
462			if value.strip().lower() not in ['y', 'yes']:
463			print('Aborting')
464			sys.exit(1)
465
466			# Prompt user for optional additional context info
467			user_info = {}
468			if not args.yes:
469			print('If you want us to get back to you via email, you can provide additional context '
470			'such as your name, email and an optional comment')
471			value = six.moves.input('Would you like to provide additional context? [y/n] ')
472			if value.strip().lower() in ['y', 'yes']:
473			user_info['name'] = six.moves.input('Name: ')
474			user_info['email'] = six.moves.input('Email: ')
475			user_info['comment'] = six.moves.input('Comment: ')
476
477			setup_logging()
478
479			if args.review:
480			create_and_review_archive(include_logs=not args.exclude_logs,
481			include_configs=not args.exclude_configs,
482			include_content=not args.exclude_content,
483			include_system_info=not args.exclude_system_info,
484			user_info=user_info,
485			debug=args.debug)
486			else:
487			create_and_upload_archive(include_logs=not args.exclude_logs,
488			include_configs=not args.exclude_configs,
489			include_content=not args.exclude_content,
490			include_system_info=not args.exclude_system_info,
491			user_info=user_info,
492			debug=args.debug)
493

StackStorm / st2

Push — master ( d1f0a7...3b2ece )

st2debug.cmd.create_archive() F

Complexity

Size

Duplication

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like