setup_pack_virtualenv() - Code Metrics - Inspection of "Allow a pack to specify Python 3 and have correspo..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#3922)

by Anthony

created 2018-01-18 03:42 UTC

setup_pack_virtualenv() F

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	1

Metric	Value
cc	13
c	2
b	0
f	1
dl	0
loc	81
rs	2.0121

How to fix Long Method Complexity

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Pack virtual environment related utility functions.
"""

import os
import re
import shutil

from oslo_config import cfg

from st2common import log as logging
from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
from st2common.constants.pack import BASE_PACK_REQUIREMENTS
from st2common.util.shell import run_command
from st2common.util.shell import quote_unix
from st2common.util.pack import get_pack_metadata
from st2common.util.compat import to_ascii
from st2common.content.utils import get_packs_base_paths
from st2common.content.utils import get_pack_directory

__all__ = [
    'setup_pack_virtualenv'
]

LOG = logging.getLogger(__name__)


def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
                          include_setuptools=True, include_wheel=True, proxy_config=None):

    """
    Setup virtual environment for the provided pack.

    :param pack_name: Name of the pack to setup the virtualenv for.
    :type pack_name: ``str``

    :param update: True to update dependencies inside the virtual environment.
    :type update: ``bool``

    :param logger: Optional logger instance to use. If not provided it defaults to the module
                   level logger.
    """
    logger = logger or LOG
    three = False  # TODO: Change default Python version to a global setting
    if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
        raise ValueError('Invalid pack name "%s"' % (pack_name))

    base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
    virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))

    # Ensure pack directory exists in one of the search paths
    pack_path = get_pack_directory(pack_name=pack_name)

    logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))

    if not pack_path:
        packs_base_paths = get_packs_base_paths()
        search_paths = ', '.join(packs_base_paths)
        msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
        raise Exception(msg)

    try:
        pack_meta = get_pack_metadata(pack_path)
        has_pack_meta = True
    except ValueError:
        # Pack is missing meta file
        has_pack_meta = False

    if has_pack_meta:
        logger.debug('Checking pack specific Python version.')
        if 'system' in pack_meta.keys() and 'python3' in pack_meta['system'].keys():
            three = bool(pack_meta['system']['python3'])
            logger.debug('Using Python %s in virtualenv' % (3 if three else 2))

    # 1. Create virtualenv if it doesn't exist
    if not update or not os.path.exists(virtualenv_path):
        # 0. Delete virtual environment if it exists
        remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)

        # 1. Create virtual environment
        logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
        create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
                          include_setuptools=include_setuptools, include_wheel=include_wheel,
                          three=three)

    # 2. Install base requirements which are common to all the packs
    logger.debug('Installing base requirements')
    for requirement in BASE_PACK_REQUIREMENTS:
        install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
                            proxy_config=proxy_config, logger=logger)

    # 3. Install pack-specific requirements
    requirements_file_path = os.path.join(pack_path, 'requirements.txt')
    has_requirements = os.path.isfile(requirements_file_path)

    if has_requirements:
        logger.debug('Installing pack specific requirements from "%s"' %
                     (requirements_file_path))
        install_requirements(virtualenv_path=virtualenv_path,
                             requirements_file_path=requirements_file_path,
                             proxy_config=proxy_config,
                             logger=logger)
    else:
        logger.debug('No pack specific requirements found')

    action = 'updated' if update else 'created'
    logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
                 (pack_name, action, virtualenv_path))


def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
                      include_wheel=True, three=False):
    """
    :param include_pip: Include pip binary and package in the newely created virtual environment.
    :type include_pip: ``bool``

    :param include_setuptools: Include setuptools binary and package in the newely created virtual
                               environment.
    :type include_setuptools: ``bool``

    :param include_wheel: Include wheel in the newely created virtual environment.
    :type include_wheel : ``bool``

    :param three: Use Python 3 binary
    :type  three: ``bool``
    """

    logger = logger or LOG

    if three:
        python_binary = cfg.CONF.actionrunner.python3_binary
    else:
        python_binary = cfg.CONF.actionrunner.python_binary
    virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
    virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts

    if not os.path.isfile(python_binary):
        raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))

    if not os.path.isfile(virtualenv_binary):
        raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))

    logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
                 (virtualenv_path, python_binary))

    cmd = [virtualenv_binary, '-p', python_binary]
    cmd.extend(virtualenv_opts)

    if not include_pip:
        cmd.append('--no-pip')

    if not include_setuptools:
        cmd.append('--no-setuptools')

    if not include_wheel:
        cmd.append('--no-wheel')

    cmd.extend([virtualenv_path])
    logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))

    try:
        exit_code, _, stderr = run_command(cmd=cmd)
    except OSError as e:
        raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
                                                             e.message))

    if exit_code != 0:
        raise Exception('Failed to create virtualenv in "%s": %s' %
                        (virtualenv_path, stderr))

    return True


def remove_virtualenv(virtualenv_path, logger=None):
    """
    Remove the provided virtualenv.
    """
    logger = logger or LOG

    if not os.path.exists(virtualenv_path):
        logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
        return True

    logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
    try:
        shutil.rmtree(virtualenv_path)
    except Exception as e:
        logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
        raise e

    return True


def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
    """
    Install requirements from a file.
    """
    logger = logger or LOG
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', '-U', '-r', requirements_file_path])
    env = get_env_for_subprocess_command()

    logger.debug('Installing requirements from file %s with command %s.',
                 requirements_file_path, ' '.join(cmd))
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        stdout = to_ascii(stdout)
        stderr = to_ascii(stderr)

        raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
                        (requirements_file_path, stdout, stderr))

    return True


def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
    """
    Install a single requirement.

    :param requirement: Requirement specifier.
    """
    logger = logger or LOG
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', requirement])
    env = get_env_for_subprocess_command()
    logger.debug('Installing requirement %s with command %s.',
                 requirement, ' '.join(cmd))
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        raise Exception('Failed to install requirement "%s": %s' %
                        (requirement, stdout))

    return True


def get_env_for_subprocess_command():
    """
    Retrieve environment to be used with the subprocess command.

    Note: We remove PYTHONPATH from the environment so the command works
    correctly with the newely created virtualenv.
    """
    env = os.environ.copy()

    if 'PYTHONPATH' in env:
        del env['PYTHONPATH']

    return env


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			"""
17			Pack virtual environment related utility functions.
18			"""
19
20			import os
21			import re
22			import shutil
23
24			from oslo_config import cfg
25
26			from st2common import log as logging
27			from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
28			from st2common.constants.pack import BASE_PACK_REQUIREMENTS
29			from st2common.util.shell import run_command
30			from st2common.util.shell import quote_unix
31			from st2common.util.pack import get_pack_metadata
32			from st2common.util.compat import to_ascii
33			from st2common.content.utils import get_packs_base_paths
34			from st2common.content.utils import get_pack_directory
35
36			__all__ = [
37			'setup_pack_virtualenv'
38			]
39
40			LOG = logging.getLogger(__name__)
41
42
43			def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
44			include_setuptools=True, include_wheel=True, proxy_config=None):
45
46			"""
47			Setup virtual environment for the provided pack.
48
49			:param pack_name: Name of the pack to setup the virtualenv for.
50			:type pack_name: ``str``
51
52			:param update: True to update dependencies inside the virtual environment.
53			:type update: ``bool``
54
55			:param logger: Optional logger instance to use. If not provided it defaults to the module
56			level logger.
57			"""
58			logger = logger or LOG
59			three = False # TODO: Change default Python version to a global setting
60			if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
61			raise ValueError('Invalid pack name "%s"' % (pack_name))
62
63			base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
64			virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
65
66			# Ensure pack directory exists in one of the search paths
67			pack_path = get_pack_directory(pack_name=pack_name)
68
69			logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))
70
71			if not pack_path:
72			packs_base_paths = get_packs_base_paths()
73			search_paths = ', '.join(packs_base_paths)
74			msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
75			raise Exception(msg)
76
77			try:
78			pack_meta = get_pack_metadata(pack_path)
79			has_pack_meta = True
80			except ValueError:
81			# Pack is missing meta file
82			has_pack_meta = False
83
84			if has_pack_meta:
85			logger.debug('Checking pack specific Python version.')
86			if 'system' in pack_meta.keys() and 'python3' in pack_meta['system'].keys():
87			three = bool(pack_meta['system']['python3'])
88			logger.debug('Using Python %s in virtualenv' % (3 if three else 2))
89
90			# 1. Create virtualenv if it doesn't exist
91			if not update or not os.path.exists(virtualenv_path):
92			# 0. Delete virtual environment if it exists
93			remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
94
95			# 1. Create virtual environment
96			logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
97			create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
98			include_setuptools=include_setuptools, include_wheel=include_wheel,
99			three=three)
100
101			# 2. Install base requirements which are common to all the packs
102			logger.debug('Installing base requirements')
103			for requirement in BASE_PACK_REQUIREMENTS:
104			install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
105			proxy_config=proxy_config, logger=logger)
106
107			# 3. Install pack-specific requirements
108			requirements_file_path = os.path.join(pack_path, 'requirements.txt')
109			has_requirements = os.path.isfile(requirements_file_path)
110
111			if has_requirements:
112			logger.debug('Installing pack specific requirements from "%s"' %
113			(requirements_file_path))
114			install_requirements(virtualenv_path=virtualenv_path,
115			requirements_file_path=requirements_file_path,
116			proxy_config=proxy_config,
117			logger=logger)
118			else:
119			logger.debug('No pack specific requirements found')
120
121			action = 'updated' if update else 'created'
122			logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
123			(pack_name, action, virtualenv_path))
124
125
126			def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
127			include_wheel=True, three=False):
128			"""
129			:param include_pip: Include pip binary and package in the newely created virtual environment.
130			:type include_pip: ``bool``
131
132			:param include_setuptools: Include setuptools binary and package in the newely created virtual
133			environment.
134			:type include_setuptools: ``bool``
135
136			:param include_wheel: Include wheel in the newely created virtual environment.
137			:type include_wheel : ``bool``
138
139			:param three: Use Python 3 binary
140			:type three: ``bool``
141			"""
142
143			logger = logger or LOG
144
145			if three:
146			python_binary = cfg.CONF.actionrunner.python3_binary
147			else:
148			python_binary = cfg.CONF.actionrunner.python_binary
149			virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
150			virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
151
152			if not os.path.isfile(python_binary):
153			raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
154
155			if not os.path.isfile(virtualenv_binary):
156			raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))
157
158			logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
159			(virtualenv_path, python_binary))
160
161			cmd = [virtualenv_binary, '-p', python_binary]
162			cmd.extend(virtualenv_opts)
163
164			if not include_pip:
165			cmd.append('--no-pip')
166
167			if not include_setuptools:
168			cmd.append('--no-setuptools')
169
170			if not include_wheel:
171			cmd.append('--no-wheel')
172
173			cmd.extend([virtualenv_path])
174			logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
175
176			try:
177			exit_code, _, stderr = run_command(cmd=cmd)
178			except OSError as e:
179			raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
180			e.message))
181
182			if exit_code != 0:
183			raise Exception('Failed to create virtualenv in "%s": %s' %
184			(virtualenv_path, stderr))
185
186			return True
187
188
189			def remove_virtualenv(virtualenv_path, logger=None):
190			"""
191			Remove the provided virtualenv.
192			"""
193			logger = logger or LOG
194
195			if not os.path.exists(virtualenv_path):
196			logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
197			return True
198
199			logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
200			try:
201			shutil.rmtree(virtualenv_path)
202			except Exception as e:
203			logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
204			raise e
205
206			return True
207
208
209			def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
210			"""
211			Install requirements from a file.
212			"""
213			logger = logger or LOG
214			pip_path = os.path.join(virtualenv_path, 'bin/pip')
215			cmd = [pip_path]
216
217			if proxy_config:
218			cert = proxy_config.get('proxy_ca_bundle_path', None)
219			https_proxy = proxy_config.get('https_proxy', None)
220			http_proxy = proxy_config.get('http_proxy', None)
221
222			if http_proxy:
223			cmd.extend(['--proxy', http_proxy])
224
225			if https_proxy:
226			cmd.extend(['--proxy', https_proxy])
227
228			if cert:
229			cmd.extend(['--cert', cert])
230
231			cmd.extend(['install', '-U', '-r', requirements_file_path])
232			env = get_env_for_subprocess_command()
233
234			logger.debug('Installing requirements from file %s with command %s.',
235			requirements_file_path, ' '.join(cmd))
236			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
237
238			if exit_code != 0:
239			stdout = to_ascii(stdout)
240			stderr = to_ascii(stderr)
241
242			raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
243			(requirements_file_path, stdout, stderr))
244
245			return True
246
247
248			def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
249			"""
250			Install a single requirement.
251
252			:param requirement: Requirement specifier.
253			"""
254			logger = logger or LOG
255			pip_path = os.path.join(virtualenv_path, 'bin/pip')
256			cmd = [pip_path]
257
258			if proxy_config:
259			cert = proxy_config.get('proxy_ca_bundle_path', None)
260			https_proxy = proxy_config.get('https_proxy', None)
261			http_proxy = proxy_config.get('http_proxy', None)
262
263			if http_proxy:
264			cmd.extend(['--proxy', http_proxy])
265
266			if https_proxy:
267			cmd.extend(['--proxy', https_proxy])
268
269			if cert:
270			cmd.extend(['--cert', cert])
271
272			cmd.extend(['install', requirement])
273			env = get_env_for_subprocess_command()
274			logger.debug('Installing requirement %s with command %s.',
275			requirement, ' '.join(cmd))
276			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
277
278			if exit_code != 0:
279			raise Exception('Failed to install requirement "%s": %s' %
280			(requirement, stdout))
281
282			return True
283
284
285			def get_env_for_subprocess_command():
286			"""
287			Retrieve environment to be used with the subprocess command.
288
289			Note: We remove PYTHONPATH from the environment so the command works
290			correctly with the newely created virtualenv.
291			"""
292			env = os.environ.copy()
293
294			if 'PYTHONPATH' in env:
295			del env['PYTHONPATH']
296
297			return env
298

StackStorm / st2

Pull Request — master (#3922)

setup_pack_virtualenv() F

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like