install_requirement() - Code Metrics - Inspection of "WIP: Proxy support for pack install" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#3556)

by Lakshmi

created 2017-07-14 15:58 UTC

install_requirement() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	3
Bugs	0	Features	0

Metric	Value
cc	6
c	3
b	0
f	0
dl	0
loc	35
rs	7.5384

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Pack virtual environment related utility functions.
"""

import os
import re
import shutil

from oslo_config import cfg

from st2common import log as logging
from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
from st2common.constants.pack import BASE_PACK_REQUIREMENTS
from st2common.util.shell import run_command
from st2common.util.shell import quote_unix
from st2common.util.compat import to_ascii
from st2common.content.utils import get_packs_base_paths
from st2common.content.utils import get_pack_directory

__all__ = [
    'setup_pack_virtualenv'
]

LOG = logging.getLogger(__name__)


def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
                          include_setuptools=True, include_wheel=True, proxy_config=None):

    """
    Setup virtual environment for the provided pack.

    :param pack_name: Name of the pack to setup the virtualenv for.
    :type pack_name: ``str``

    :param update: True to update dependencies inside the virtual environment.
    :type update: ``bool``

    :param logger: Optional logger instance to use. If not provided it defaults to the module
                   level logger.
    """
    logger = logger or LOG

    if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
        raise ValueError('Invalid pack name "%s"' % (pack_name))

    base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
    virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))

    # Ensure pack directory exists in one of the search paths
    pack_path = get_pack_directory(pack_name=pack_name)

    logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))

    if not pack_path:
        packs_base_paths = get_packs_base_paths()
        search_paths = ', '.join(packs_base_paths)
        msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
        raise Exception(msg)

    # 1. Create virtualenv if it doesn't exist
    if not update or not os.path.exists(virtualenv_path):
        # 0. Delete virtual environment if it exists
        remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)

        # 1. Create virtual environment
        logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
        create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
                          include_setuptools=include_setuptools, include_wheel=include_wheel)

    # 2. Install base requirements which are common to all the packs
    logger.debug('Installing base requirements')
    for requirement in BASE_PACK_REQUIREMENTS:
        install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
                            proxy_config=proxy_config, logger=logger)

    # 3. Install pack-specific requirements
    requirements_file_path = os.path.join(pack_path, 'requirements.txt')
    has_requirements = os.path.isfile(requirements_file_path)

    if has_requirements:
        logger.debug('Installing pack specific requirements from "%s"' %
                     (requirements_file_path))
        install_requirements(virtualenv_path=virtualenv_path,
                             requirements_file_path=requirements_file_path,
                             proxy_config=proxy_config,
                             logger=logger)
    else:
        logger.debug('No pack specific requirements found')

    action = 'updated' if update else 'created'
    logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
                 (pack_name, action, virtualenv_path))


def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
                      include_wheel=True):
    """
    :param include_pip: Include pip binary and package in the newely created virtual environment.
    :type include_pip: ``bool``

    :param include_setuptools: Include setuptools binary and package in the newely created virtual
                               environment.
    :type include_setuptools: ``bool``

    :param include_wheel: Include wheel in the newely created virtual environment.
    :type include_wheel : ``bool``
    """

    logger = logger or LOG

    python_binary = cfg.CONF.actionrunner.python_binary
    virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
    virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts

    if not os.path.isfile(python_binary):
        raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))

    if not os.path.isfile(virtualenv_binary):
        raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))

    logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
                 (virtualenv_path, python_binary))

    cmd = [virtualenv_binary, '-p', python_binary]
    cmd.extend(virtualenv_opts)

    if not include_pip:
        cmd.append('--no-pip')

    if not include_setuptools:
        cmd.append('--no-setuptools')

    if not include_wheel:
        cmd.append('--no-wheel')

    cmd.extend([virtualenv_path])
    logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))

    try:
        exit_code, _, stderr = run_command(cmd=cmd)
    except OSError as e:
        raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
                                                             e.message))

    if exit_code != 0:
        raise Exception('Failed to create virtualenv in "%s": %s' %
                        (virtualenv_path, stderr))

    return True


def remove_virtualenv(virtualenv_path, logger=None):
    """
    Remove the provided virtualenv.
    """
    logger = logger or LOG

    if not os.path.exists(virtualenv_path):
        logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
        return True

    logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
    try:
        shutil.rmtree(virtualenv_path)
    except Exception as e:
        logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
        raise e

    return True


def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
    """
    Install requirements from a file.
    """
    logger = logger or LOG
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', '-U', '-r', requirements_file_path])
    env = get_env_for_subprocess_command()

    logger.debug('Installing requirements from file %s with command %s.',
                 requirements_file_path, ' '.join(cmd))
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        stdout = to_ascii(stdout)
        stderr = to_ascii(stderr)

        raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
                        (requirements_file_path, stdout, stderr))

    return True


def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
    """
    Install a single requirement.

    :param requirement: Requirement specifier.
    """
    logger = logger or LOG
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', requirement])
    env = get_env_for_subprocess_command()
    logger.debug('Installing requirement %s with command %s.',
                 requirement, ' '.join(cmd))
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        raise Exception('Failed to install requirement "%s": %s' %
                        (requirement, stdout))

    return True


def get_env_for_subprocess_command():
    """
    Retrieve environment to be used with the subprocess command.

    Note: We remove PYTHONPATH from the environment so the command works
    correctly with the newely created virtualenv.
    """
    env = os.environ.copy()

    if 'PYTHONPATH' in env:
        del env['PYTHONPATH']

    return env


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			"""
17			Pack virtual environment related utility functions.
18			"""
19
20			import os
21			import re
22			import shutil
23
24			from oslo_config import cfg
25
26			from st2common import log as logging
27			from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
28			from st2common.constants.pack import BASE_PACK_REQUIREMENTS
29			from st2common.util.shell import run_command
30			from st2common.util.shell import quote_unix
31			from st2common.util.compat import to_ascii
32			from st2common.content.utils import get_packs_base_paths
33			from st2common.content.utils import get_pack_directory
34
35			__all__ = [
36			'setup_pack_virtualenv'
37			]
38
39			LOG = logging.getLogger(__name__)
40
41
42			def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
43			include_setuptools=True, include_wheel=True, proxy_config=None):
44
45			"""
46			Setup virtual environment for the provided pack.
47
48			:param pack_name: Name of the pack to setup the virtualenv for.
49			:type pack_name: ``str``
50
51			:param update: True to update dependencies inside the virtual environment.
52			:type update: ``bool``
53
54			:param logger: Optional logger instance to use. If not provided it defaults to the module
55			level logger.
56			"""
57			logger = logger or LOG
58
59			if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
60			raise ValueError('Invalid pack name "%s"' % (pack_name))
61
62			base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
63			virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
64
65			# Ensure pack directory exists in one of the search paths
66			pack_path = get_pack_directory(pack_name=pack_name)
67
68			logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))
69
70			if not pack_path:
71			packs_base_paths = get_packs_base_paths()
72			search_paths = ', '.join(packs_base_paths)
73			msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
74			raise Exception(msg)
75
76			# 1. Create virtualenv if it doesn't exist
77			if not update or not os.path.exists(virtualenv_path):
78			# 0. Delete virtual environment if it exists
79			remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
80
81			# 1. Create virtual environment
82			logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
83			create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
84			include_setuptools=include_setuptools, include_wheel=include_wheel)
85
86			# 2. Install base requirements which are common to all the packs
87			logger.debug('Installing base requirements')
88			for requirement in BASE_PACK_REQUIREMENTS:
89			install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
90			proxy_config=proxy_config, logger=logger)
91
92			# 3. Install pack-specific requirements
93			requirements_file_path = os.path.join(pack_path, 'requirements.txt')
94			has_requirements = os.path.isfile(requirements_file_path)
95
96			if has_requirements:
97			logger.debug('Installing pack specific requirements from "%s"' %
98			(requirements_file_path))
99			install_requirements(virtualenv_path=virtualenv_path,
100			requirements_file_path=requirements_file_path,
101			proxy_config=proxy_config,
102			logger=logger)
103			else:
104			logger.debug('No pack specific requirements found')
105
106			action = 'updated' if update else 'created'
107			logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
108			(pack_name, action, virtualenv_path))
109
110
111			def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
112			include_wheel=True):
113			"""
114			:param include_pip: Include pip binary and package in the newely created virtual environment.
115			:type include_pip: ``bool``
116
117			:param include_setuptools: Include setuptools binary and package in the newely created virtual
118			environment.
119			:type include_setuptools: ``bool``
120
121			:param include_wheel: Include wheel in the newely created virtual environment.
122			:type include_wheel : ``bool``
123			"""
124
125			logger = logger or LOG
126
127			python_binary = cfg.CONF.actionrunner.python_binary
128			virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
129			virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
130
131			if not os.path.isfile(python_binary):
132			raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
133
134			if not os.path.isfile(virtualenv_binary):
135			raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))
136
137			logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
138			(virtualenv_path, python_binary))
139
140			cmd = [virtualenv_binary, '-p', python_binary]
141			cmd.extend(virtualenv_opts)
142
143			if not include_pip:
144			cmd.append('--no-pip')
145
146			if not include_setuptools:
147			cmd.append('--no-setuptools')
148
149			if not include_wheel:
150			cmd.append('--no-wheel')
151
152			cmd.extend([virtualenv_path])
153			logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
154
155			try:
156			exit_code, _, stderr = run_command(cmd=cmd)
157			except OSError as e:
158			raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
159			e.message))
160
161			if exit_code != 0:
162			raise Exception('Failed to create virtualenv in "%s": %s' %
163			(virtualenv_path, stderr))
164
165			return True
166
167
168			def remove_virtualenv(virtualenv_path, logger=None):
169			"""
170			Remove the provided virtualenv.
171			"""
172			logger = logger or LOG
173
174			if not os.path.exists(virtualenv_path):
175			logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
176			return True
177
178			logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
179			try:
180			shutil.rmtree(virtualenv_path)
181			except Exception as e:
182			logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
183			raise e
184
185			return True
186
187
188			def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
189			"""
190			Install requirements from a file.
191			"""
192			logger = logger or LOG
193			pip_path = os.path.join(virtualenv_path, 'bin/pip')
194			cmd = [pip_path]
195
196			if proxy_config:
197			cert = proxy_config.get('proxy_ca_bundle_path', None)
198			https_proxy = proxy_config.get('https_proxy', None)
199			http_proxy = proxy_config.get('http_proxy', None)
200
201			if http_proxy:
202			cmd.extend(['--proxy', http_proxy])
203
204			if https_proxy:
205			cmd.extend(['--proxy', https_proxy])
206
207			if cert:
208			cmd.extend(['--cert', cert])
209
210			cmd.extend(['install', '-U', '-r', requirements_file_path])
211			env = get_env_for_subprocess_command()
212
213			logger.debug('Installing requirements from file %s with command %s.',
214			requirements_file_path, ' '.join(cmd))
215			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
216
217			if exit_code != 0:
218			stdout = to_ascii(stdout)
219			stderr = to_ascii(stderr)
220
221			raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
222			(requirements_file_path, stdout, stderr))
223
224			return True
225
226
227			def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
228			"""
229			Install a single requirement.
230
231			:param requirement: Requirement specifier.
232			"""
233			logger = logger or LOG
234			pip_path = os.path.join(virtualenv_path, 'bin/pip')
235			cmd = [pip_path]
236
237			if proxy_config:
238			cert = proxy_config.get('proxy_ca_bundle_path', None)
239			https_proxy = proxy_config.get('https_proxy', None)
240			http_proxy = proxy_config.get('http_proxy', None)
241
242			if http_proxy:
243			cmd.extend(['--proxy', http_proxy])
244
245			if https_proxy:
246			cmd.extend(['--proxy', https_proxy])
247
248			if cert:
249			cmd.extend(['--cert', cert])
250
251			cmd.extend(['install', requirement])
252			env = get_env_for_subprocess_command()
253			logger.debug('Installing requirement %s with command %s.',
254			requirement, ' '.join(cmd))
255			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
256
257			if exit_code != 0:
258			raise Exception('Failed to install requirement "%s": %s' %
259			(requirement, stdout))
260
261			return True
262
263
264			def get_env_for_subprocess_command():
265			"""
266			Retrieve environment to be used with the subprocess command.
267
268			Note: We remove PYTHONPATH from the environment so the command works
269			correctly with the newely created virtualenv.
270			"""
271			env = os.environ.copy()
272
273			if 'PYTHONPATH' in env:
274			del env['PYTHONPATH']
275
276			return env
277

StackStorm / st2

Pull Request — master (#3556)

install_requirement() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like