create_virtualenv() - Code Metrics - Inspection of "[WIP] Allow a pack (or the user) to specify the Py..." - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#3922)

by Anthony

created 2018-01-01 03:43 UTC

create_virtualenv() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	1

Metric	Value
cc	9
dl	0
loc	61
rs	4.8709
c	1
b	0
f	1

How to fix Long Method

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Pack virtual environment related utility functions.
"""

import os
import re
import shutil

from oslo_config import cfg

from st2common import log as logging
from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
from st2common.constants.pack import BASE_PACK_REQUIREMENTS
from st2common.util.shell import run_command
from st2common.util.shell import quote_unix
from st2common.util.compat import to_ascii
from st2common.content.utils import get_packs_base_paths
from st2common.content.utils import get_pack_directory

__all__ = [
    'setup_pack_virtualenv'
]

LOG = logging.getLogger(__name__)


def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
                          include_setuptools=True, include_wheel=True, proxy_config=None):

    """
    Setup virtual environment for the provided pack.

    :param pack_name: Name of the pack to setup the virtualenv for.
    :type pack_name: ``str``

    :param update: True to update dependencies inside the virtual environment.
    :type update: ``bool``

    :param logger: Optional logger instance to use. If not provided it defaults to the module
                   level logger.
    """
    logger = logger or LOG
    three = False  # TODO: Either pull Python 3 flag from pack settings or a global env
    if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
        raise ValueError('Invalid pack name "%s"' % (pack_name))

    base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
    virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))

    # Ensure pack directory exists in one of the search paths
    pack_path = get_pack_directory(pack_name=pack_name)

    logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))

    if not pack_path:
        packs_base_paths = get_packs_base_paths()
        search_paths = ', '.join(packs_base_paths)
        msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
        raise Exception(msg)

    # 1. Create virtualenv if it doesn't exist
    if not update or not os.path.exists(virtualenv_path):
        # 0. Delete virtual environment if it exists
        remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)

        # 1. Create virtual environment
        logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
        create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
                          include_setuptools=include_setuptools, include_wheel=include_wheel,
                          three=three)

    # 2. Install base requirements which are common to all the packs
    logger.debug('Installing base requirements')
    for requirement in BASE_PACK_REQUIREMENTS:
        install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
                            proxy_config=proxy_config, logger=logger)

    # 3. Install pack-specific requirements
    requirements_file_path = os.path.join(pack_path, 'requirements.txt')
    has_requirements = os.path.isfile(requirements_file_path)

    if has_requirements:
        logger.debug('Installing pack specific requirements from "%s"' %
                     (requirements_file_path))
        install_requirements(virtualenv_path=virtualenv_path,
                             requirements_file_path=requirements_file_path,
                             proxy_config=proxy_config,
                             logger=logger)
    else:
        logger.debug('No pack specific requirements found')

    action = 'updated' if update else 'created'
    logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
                 (pack_name, action, virtualenv_path))


def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
                      include_wheel=True, three=False):
    """
    :param include_pip: Include pip binary and package in the newely created virtual environment.
    :type include_pip: ``bool``

    :param include_setuptools: Include setuptools binary and package in the newely created virtual
                               environment.
    :type include_setuptools: ``bool``

    :param include_wheel: Include wheel in the newely created virtual environment.
    :type include_wheel : ``bool``

    :param three: Use Python 3 binary
    :type  three: ``bool``
    """

    logger = logger or LOG

    if three:
        python_binary = cfg.CONF.actionrunner.python_binary
    else:
        python_binary = cfg.CONF.actionrunner.python3_binary
    virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
    virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts

    if not os.path.isfile(python_binary):
        raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))

    if not os.path.isfile(virtualenv_binary):
        raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))

    logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
                 (virtualenv_path, python_binary))

    cmd = [virtualenv_binary, '-p', python_binary]
    cmd.extend(virtualenv_opts)

    if not include_pip:
        cmd.append('--no-pip')

    if not include_setuptools:
        cmd.append('--no-setuptools')

    if not include_wheel:
        cmd.append('--no-wheel')

    cmd.extend([virtualenv_path])
    logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))

    try:
        exit_code, _, stderr = run_command(cmd=cmd)
    except OSError as e:
        raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
                                                             e.message))

    if exit_code != 0:
        raise Exception('Failed to create virtualenv in "%s": %s' %
                        (virtualenv_path, stderr))

    return True


def remove_virtualenv(virtualenv_path, logger=None):
    """
    Remove the provided virtualenv.
    """
    logger = logger or LOG

    if not os.path.exists(virtualenv_path):
        logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
        return True

    logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
    try:
        shutil.rmtree(virtualenv_path)
    except Exception as e:
        logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
        raise e

    return True


def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
    """
    Install requirements from a file.
    """
    logger = logger or LOG
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', '-U', '-r', requirements_file_path])
    env = get_env_for_subprocess_command()

    logger.debug('Installing requirements from file %s with command %s.',
                 requirements_file_path, ' '.join(cmd))
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        stdout = to_ascii(stdout)
        stderr = to_ascii(stderr)

        raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
                        (requirements_file_path, stdout, stderr))

    return True


def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
    """
    Install a single requirement.

    :param requirement: Requirement specifier.
    """
    logger = logger or LOG
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', requirement])
    env = get_env_for_subprocess_command()
    logger.debug('Installing requirement %s with command %s.',
                 requirement, ' '.join(cmd))
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        raise Exception('Failed to install requirement "%s": %s' %
                        (requirement, stdout))

    return True


def get_env_for_subprocess_command():
    """
    Retrieve environment to be used with the subprocess command.

    Note: We remove PYTHONPATH from the environment so the command works
    correctly with the newely created virtualenv.
    """
    env = os.environ.copy()

    if 'PYTHONPATH' in env:
        del env['PYTHONPATH']

    return env


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			"""
17			Pack virtual environment related utility functions.
18			"""
19
20			import os
21			import re
22			import shutil
23
24			from oslo_config import cfg
25
26			from st2common import log as logging
27			from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
28			from st2common.constants.pack import BASE_PACK_REQUIREMENTS
29			from st2common.util.shell import run_command
30			from st2common.util.shell import quote_unix
31			from st2common.util.compat import to_ascii
32			from st2common.content.utils import get_packs_base_paths
33			from st2common.content.utils import get_pack_directory
34
35			__all__ = [
36			'setup_pack_virtualenv'
37			]
38
39			LOG = logging.getLogger(__name__)
40
41
42			def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
43			include_setuptools=True, include_wheel=True, proxy_config=None):
44
45			"""
46			Setup virtual environment for the provided pack.
47
48			:param pack_name: Name of the pack to setup the virtualenv for.
49			:type pack_name: ``str``
50
51			:param update: True to update dependencies inside the virtual environment.
52			:type update: ``bool``
53
54			:param logger: Optional logger instance to use. If not provided it defaults to the module
55			level logger.
56			"""
57			logger = logger or LOG
58			three = False # TODO: Either pull Python 3 flag from pack settings or a global env
59			if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
60			raise ValueError('Invalid pack name "%s"' % (pack_name))
61
62			base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
63			virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
64
65			# Ensure pack directory exists in one of the search paths
66			pack_path = get_pack_directory(pack_name=pack_name)
67
68			logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))
69
70			if not pack_path:
71			packs_base_paths = get_packs_base_paths()
72			search_paths = ', '.join(packs_base_paths)
73			msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
74			raise Exception(msg)
75
76			# 1. Create virtualenv if it doesn't exist
77			if not update or not os.path.exists(virtualenv_path):
78			# 0. Delete virtual environment if it exists
79			remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
80
81			# 1. Create virtual environment
82			logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
83			create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
84			include_setuptools=include_setuptools, include_wheel=include_wheel,
85			three=three)
86
87			# 2. Install base requirements which are common to all the packs
88			logger.debug('Installing base requirements')
89			for requirement in BASE_PACK_REQUIREMENTS:
90			install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
91			proxy_config=proxy_config, logger=logger)
92
93			# 3. Install pack-specific requirements
94			requirements_file_path = os.path.join(pack_path, 'requirements.txt')
95			has_requirements = os.path.isfile(requirements_file_path)
96
97			if has_requirements:
98			logger.debug('Installing pack specific requirements from "%s"' %
99			(requirements_file_path))
100			install_requirements(virtualenv_path=virtualenv_path,
101			requirements_file_path=requirements_file_path,
102			proxy_config=proxy_config,
103			logger=logger)
104			else:
105			logger.debug('No pack specific requirements found')
106
107			action = 'updated' if update else 'created'
108			logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
109			(pack_name, action, virtualenv_path))
110
111
112			def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
113			include_wheel=True, three=False):
114			"""
115			:param include_pip: Include pip binary and package in the newely created virtual environment.
116			:type include_pip: ``bool``
117
118			:param include_setuptools: Include setuptools binary and package in the newely created virtual
119			environment.
120			:type include_setuptools: ``bool``
121
122			:param include_wheel: Include wheel in the newely created virtual environment.
123			:type include_wheel : ``bool``
124
125			:param three: Use Python 3 binary
126			:type three: ``bool``
127			"""
128
129			logger = logger or LOG
130
131			if three:
132			python_binary = cfg.CONF.actionrunner.python_binary
133			else:
134			python_binary = cfg.CONF.actionrunner.python3_binary
135			virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
136			virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
137
138			if not os.path.isfile(python_binary):
139			raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
140
141			if not os.path.isfile(virtualenv_binary):
142			raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))
143
144			logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
145			(virtualenv_path, python_binary))
146
147			cmd = [virtualenv_binary, '-p', python_binary]
148			cmd.extend(virtualenv_opts)
149
150			if not include_pip:
151			cmd.append('--no-pip')
152
153			if not include_setuptools:
154			cmd.append('--no-setuptools')
155
156			if not include_wheel:
157			cmd.append('--no-wheel')
158
159			cmd.extend([virtualenv_path])
160			logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
161
162			try:
163			exit_code, _, stderr = run_command(cmd=cmd)
164			except OSError as e:
165			raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
166			e.message))
167
168			if exit_code != 0:
169			raise Exception('Failed to create virtualenv in "%s": %s' %
170			(virtualenv_path, stderr))
171
172			return True
173
174
175			def remove_virtualenv(virtualenv_path, logger=None):
176			"""
177			Remove the provided virtualenv.
178			"""
179			logger = logger or LOG
180
181			if not os.path.exists(virtualenv_path):
182			logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
183			return True
184
185			logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
186			try:
187			shutil.rmtree(virtualenv_path)
188			except Exception as e:
189			logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
190			raise e
191
192			return True
193
194
195			def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
196			"""
197			Install requirements from a file.
198			"""
199			logger = logger or LOG
200			pip_path = os.path.join(virtualenv_path, 'bin/pip')
201			cmd = [pip_path]
202
203			if proxy_config:
204			cert = proxy_config.get('proxy_ca_bundle_path', None)
205			https_proxy = proxy_config.get('https_proxy', None)
206			http_proxy = proxy_config.get('http_proxy', None)
207
208			if http_proxy:
209			cmd.extend(['--proxy', http_proxy])
210
211			if https_proxy:
212			cmd.extend(['--proxy', https_proxy])
213
214			if cert:
215			cmd.extend(['--cert', cert])
216
217			cmd.extend(['install', '-U', '-r', requirements_file_path])
218			env = get_env_for_subprocess_command()
219
220			logger.debug('Installing requirements from file %s with command %s.',
221			requirements_file_path, ' '.join(cmd))
222			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
223
224			if exit_code != 0:
225			stdout = to_ascii(stdout)
226			stderr = to_ascii(stderr)
227
228			raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
229			(requirements_file_path, stdout, stderr))
230
231			return True
232
233
234			def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
235			"""
236			Install a single requirement.
237
238			:param requirement: Requirement specifier.
239			"""
240			logger = logger or LOG
241			pip_path = os.path.join(virtualenv_path, 'bin/pip')
242			cmd = [pip_path]
243
244			if proxy_config:
245			cert = proxy_config.get('proxy_ca_bundle_path', None)
246			https_proxy = proxy_config.get('https_proxy', None)
247			http_proxy = proxy_config.get('http_proxy', None)
248
249			if http_proxy:
250			cmd.extend(['--proxy', http_proxy])
251
252			if https_proxy:
253			cmd.extend(['--proxy', https_proxy])
254
255			if cert:
256			cmd.extend(['--cert', cert])
257
258			cmd.extend(['install', requirement])
259			env = get_env_for_subprocess_command()
260			logger.debug('Installing requirement %s with command %s.',
261			requirement, ' '.join(cmd))
262			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
263
264			if exit_code != 0:
265			raise Exception('Failed to install requirement "%s": %s' %
266			(requirement, stdout))
267
268			return True
269
270
271			def get_env_for_subprocess_command():
272			"""
273			Retrieve environment to be used with the subprocess command.
274
275			Note: We remove PYTHONPATH from the environment so the command works
276			correctly with the newely created virtualenv.
277			"""
278			env = os.environ.copy()
279
280			if 'PYTHONPATH' in env:
281			del env['PYTHONPATH']
282
283			return env
284

StackStorm / st2

Pull Request — master (#3922)

create_virtualenv() D

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like