install_requirement() - Code Metrics - Inspection of "WIP: Proxy support for pack install" - StackStorm/st2 - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#3556)

by Lakshmi

created 2017-07-13 17:43 UTC

install_requirement() B

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	6
dl	0
loc	32
rs	7.5384
c	2
b	0
f	0

# Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Pack virtual environment related utility functions.
"""

import os
import re
import shutil

from oslo_config import cfg

from st2common import log as logging
from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
from st2common.constants.pack import BASE_PACK_REQUIREMENTS
from st2common.util.shell import run_command
from st2common.util.shell import quote_unix
from st2common.util.compat import to_ascii
from st2common.content.utils import get_packs_base_paths
from st2common.content.utils import get_pack_directory

__all__ = [
    'setup_pack_virtualenv'
]

LOG = logging.getLogger(__name__)


def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
                          include_setuptools=True, include_wheel=True, proxy_config=None):

    """
    Setup virtual environment for the provided pack.

    :param pack_name: Name of the pack to setup the virtualenv for.
    :type pack_name: ``str``

    :param update: True to update dependencies inside the virtual environment.
    :type update: ``bool``

    :param logger: Optional logger instance to use. If not provided it defaults to the module
                   level logger.
    """
    logger = logger or LOG

    if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
        raise ValueError('Invalid pack name "%s"' % (pack_name))

    base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
    virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))

    # Ensure pack directory exists in one of the search paths
    pack_path = get_pack_directory(pack_name=pack_name)

    logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))

    if not pack_path:
        packs_base_paths = get_packs_base_paths()
        search_paths = ', '.join(packs_base_paths)
        msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
        raise Exception(msg)

    # 1. Create virtualenv if it doesn't exist
    if not update or not os.path.exists(virtualenv_path):
        # 0. Delete virtual environment if it exists
        remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)

        # 1. Create virtual environment
        logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
        create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
                          include_setuptools=include_setuptools, include_wheel=include_wheel)

    # 2. Install base requirements which are common to all the packs
    logger.debug('Installing base requirements')
    for requirement in BASE_PACK_REQUIREMENTS:
        install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
                            proxy_config=proxy_config)

    # 3. Install pack-specific requirements
    requirements_file_path = os.path.join(pack_path, 'requirements.txt')
    has_requirements = os.path.isfile(requirements_file_path)

    if has_requirements:
        logger.debug('Installing pack specific requirements from "%s"' %
                     (requirements_file_path))
        install_requirements(virtualenv_path=virtualenv_path,
                             requirements_file_path=requirements_file_path,
                             proxy_config=proxy_config)
    else:
        logger.debug('No pack specific requirements found')

    action = 'updated' if update else 'created'
    logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
                 (pack_name, action, virtualenv_path))


def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
                      include_wheel=True):
    """
    :param include_pip: Include pip binary and package in the newely created virtual environment.
    :type include_pip: ``bool``

    :param include_setuptools: Include setuptools binary and package in the newely created virtual
                               environment.
    :type include_setuptools: ``bool``

    :param include_wheel: Include wheel in the newely created virtual environment.
    :type include_wheel : ``bool``
    """

    logger = logger or LOG

    python_binary = cfg.CONF.actionrunner.python_binary
    virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
    virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts

    if not os.path.isfile(python_binary):
        raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))

    if not os.path.isfile(virtualenv_binary):
        raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))

    logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
                 (virtualenv_path, python_binary))

    cmd = [virtualenv_binary, '-p', python_binary]
    cmd.extend(virtualenv_opts)

    if not include_pip:
        cmd.append('--no-pip')

    if not include_setuptools:
        cmd.append('--no-setuptools')

    if not include_wheel:
        cmd.append('--no-wheel')

    cmd.extend([virtualenv_path])
    logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))

    try:
        exit_code, _, stderr = run_command(cmd=cmd)
    except OSError as e:
        raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
                                                             e.message))

    if exit_code != 0:
        raise Exception('Failed to create virtualenv in "%s": %s' %
                        (virtualenv_path, stderr))

    return True


def remove_virtualenv(virtualenv_path, logger=None):
    """
    Remove the provided virtualenv.
    """
    logger = logger or LOG

    if not os.path.exists(virtualenv_path):
        logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
        return True

    logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
    try:
        shutil.rmtree(virtualenv_path)
    except Exception as e:
        logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
        raise e

    return True


def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None):
    """
    Install requirements from a file.
    """
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', '-U', '-r', requirements_file_path])
    env = get_env_for_subprocess_command()

    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        stdout = to_ascii(stdout)
        stderr = to_ascii(stderr)

        raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
                        (requirements_file_path, stdout, stderr))

    return True


def install_requirement(virtualenv_path, requirement, proxy_config=None):
    """
    Install a single requirement.

    :param requirement: Requirement specifier.
    """
    pip_path = os.path.join(virtualenv_path, 'bin/pip')
    cmd = [pip_path]

    if proxy_config:
        cert = proxy_config.get('proxy_ca_bundle_path', None)
        https_proxy = proxy_config.get('https_proxy', None)
        http_proxy = proxy_config.get('http_proxy', None)

        if http_proxy:
            cmd.extend(['--proxy', http_proxy])

        if https_proxy:
            cmd.extend(['--proxy', https_proxy])

        if cert:
            cmd.extend(['--cert', cert])

    cmd.extend(['install', requirement])
    env = get_env_for_subprocess_command()
    exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

    if exit_code != 0:
        raise Exception('Failed to install requirement "%s": %s' %
                        (requirement, stdout))

    return True


def get_env_for_subprocess_command():
    """
    Retrieve environment to be used with the subprocess command.

    Note: We remove PYTHONPATH from the environment so the command works
    correctly with the newely created virtualenv.
    """
    env = os.environ.copy()

    if 'PYTHONPATH' in env:
        del env['PYTHONPATH']

    return env


1			# Licensed to the StackStorm, Inc ('StackStorm') under one or more
2			# contributor license agreements. See the NOTICE file distributed with
3			# this work for additional information regarding copyright ownership.
4			# The ASF licenses this file to You under the Apache License, Version 2.0
5			# (the "License"); you may not use this file except in compliance with
6			# the License. You may obtain a copy of the License at
7			#
8			# http://www.apache.org/licenses/LICENSE-2.0
9			#
10			# Unless required by applicable law or agreed to in writing, software
11			# distributed under the License is distributed on an "AS IS" BASIS,
12			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13			# See the License for the specific language governing permissions and
14			# limitations under the License.
15
16			"""
17			Pack virtual environment related utility functions.
18			"""
19
20			import os
21			import re
22			import shutil
23
24			from oslo_config import cfg
25
26			from st2common import log as logging
27			from st2common.constants.pack import PACK_REF_WHITELIST_REGEX
28			from st2common.constants.pack import BASE_PACK_REQUIREMENTS
29			from st2common.util.shell import run_command
30			from st2common.util.shell import quote_unix
31			from st2common.util.compat import to_ascii
32			from st2common.content.utils import get_packs_base_paths
33			from st2common.content.utils import get_pack_directory
34
35			__all__ = [
36			'setup_pack_virtualenv'
37			]
38
39			LOG = logging.getLogger(__name__)
40
41
42			def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
43			include_setuptools=True, include_wheel=True, proxy_config=None):
44
45			"""
46			Setup virtual environment for the provided pack.
47
48			:param pack_name: Name of the pack to setup the virtualenv for.
49			:type pack_name: ``str``
50
51			:param update: True to update dependencies inside the virtual environment.
52			:type update: ``bool``
53
54			:param logger: Optional logger instance to use. If not provided it defaults to the module
55			level logger.
56			"""
57			logger = logger or LOG
58
59			if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
60			raise ValueError('Invalid pack name "%s"' % (pack_name))
61
62			base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
63			virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
64
65			# Ensure pack directory exists in one of the search paths
66			pack_path = get_pack_directory(pack_name=pack_name)
67
68			logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path))
69
70			if not pack_path:
71			packs_base_paths = get_packs_base_paths()
72			search_paths = ', '.join(packs_base_paths)
73			msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
74			raise Exception(msg)
75
76			# 1. Create virtualenv if it doesn't exist
77			if not update or not os.path.exists(virtualenv_path):
78			# 0. Delete virtual environment if it exists
79			remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
80
81			# 1. Create virtual environment
82			logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
83			create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
84			include_setuptools=include_setuptools, include_wheel=include_wheel)
85
86			# 2. Install base requirements which are common to all the packs
87			logger.debug('Installing base requirements')
88			for requirement in BASE_PACK_REQUIREMENTS:
89			install_requirement(virtualenv_path=virtualenv_path, requirement=requirement,
90			proxy_config=proxy_config)
91
92			# 3. Install pack-specific requirements
93			requirements_file_path = os.path.join(pack_path, 'requirements.txt')
94			has_requirements = os.path.isfile(requirements_file_path)
95
96			if has_requirements:
97			logger.debug('Installing pack specific requirements from "%s"' %
98			(requirements_file_path))
99			install_requirements(virtualenv_path=virtualenv_path,
100			requirements_file_path=requirements_file_path,
101			proxy_config=proxy_config)
102			else:
103			logger.debug('No pack specific requirements found')
104
105			action = 'updated' if update else 'created'
106			logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
107			(pack_name, action, virtualenv_path))
108
109
110			def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
111			include_wheel=True):
112			"""
113			:param include_pip: Include pip binary and package in the newely created virtual environment.
114			:type include_pip: ``bool``
115
116			:param include_setuptools: Include setuptools binary and package in the newely created virtual
117			environment.
118			:type include_setuptools: ``bool``
119
120			:param include_wheel: Include wheel in the newely created virtual environment.
121			:type include_wheel : ``bool``
122			"""
123
124			logger = logger or LOG
125
126			python_binary = cfg.CONF.actionrunner.python_binary
127			virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
128			virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
129
130			if not os.path.isfile(python_binary):
131			raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
132
133			if not os.path.isfile(virtualenv_binary):
134			raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))
135
136			logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
137			(virtualenv_path, python_binary))
138
139			cmd = [virtualenv_binary, '-p', python_binary]
140			cmd.extend(virtualenv_opts)
141
142			if not include_pip:
143			cmd.append('--no-pip')
144
145			if not include_setuptools:
146			cmd.append('--no-setuptools')
147
148			if not include_wheel:
149			cmd.append('--no-wheel')
150
151			cmd.extend([virtualenv_path])
152			logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
153
154			try:
155			exit_code, _, stderr = run_command(cmd=cmd)
156			except OSError as e:
157			raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
158			e.message))
159
160			if exit_code != 0:
161			raise Exception('Failed to create virtualenv in "%s": %s' %
162			(virtualenv_path, stderr))
163
164			return True
165
166
167			def remove_virtualenv(virtualenv_path, logger=None):
168			"""
169			Remove the provided virtualenv.
170			"""
171			logger = logger or LOG
172
173			if not os.path.exists(virtualenv_path):
174			logger.info('Virtualenv path "%s" doesn\'t exist' % virtualenv_path)
175			return True
176
177			logger.debug('Removing virtualenv in "%s"' % virtualenv_path)
178			try:
179			shutil.rmtree(virtualenv_path)
180			except Exception as e:
181			logger.error('Error while removing virtualenv at "%s": "%s"' % (virtualenv_path, e))
182			raise e
183
184			return True
185
186
187			def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None):
188			"""
189			Install requirements from a file.
190			"""
191			pip_path = os.path.join(virtualenv_path, 'bin/pip')
192			cmd = [pip_path]
193
194			if proxy_config:
195			cert = proxy_config.get('proxy_ca_bundle_path', None)
196			https_proxy = proxy_config.get('https_proxy', None)
197			http_proxy = proxy_config.get('http_proxy', None)
198
199			if http_proxy:
200			cmd.extend(['--proxy', http_proxy])
201
202			if https_proxy:
203			cmd.extend(['--proxy', https_proxy])
204
205			if cert:
206			cmd.extend(['--cert', cert])
207
208			cmd.extend(['install', '-U', '-r', requirements_file_path])
209			env = get_env_for_subprocess_command()
210
211			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
212
213			if exit_code != 0:
214			stdout = to_ascii(stdout)
215			stderr = to_ascii(stderr)
216
217			raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
218			(requirements_file_path, stdout, stderr))
219
220			return True
221
222
223			def install_requirement(virtualenv_path, requirement, proxy_config=None):
224			"""
225			Install a single requirement.
226
227			:param requirement: Requirement specifier.
228			"""
229			pip_path = os.path.join(virtualenv_path, 'bin/pip')
230			cmd = [pip_path]
231
232			if proxy_config:
233			cert = proxy_config.get('proxy_ca_bundle_path', None)
234			https_proxy = proxy_config.get('https_proxy', None)
235			http_proxy = proxy_config.get('http_proxy', None)
236
237			if http_proxy:
238			cmd.extend(['--proxy', http_proxy])
239
240			if https_proxy:
241			cmd.extend(['--proxy', https_proxy])
242
243			if cert:
244			cmd.extend(['--cert', cert])
245
246			cmd.extend(['install', requirement])
247			env = get_env_for_subprocess_command()
248			exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
249
250			if exit_code != 0:
251			raise Exception('Failed to install requirement "%s": %s' %
252			(requirement, stdout))
253
254			return True
255
256
257			def get_env_for_subprocess_command():
258			"""
259			Retrieve environment to be used with the subprocess command.
260
261			Note: We remove PYTHONPATH from the environment so the command works
262			correctly with the newely created virtualenv.
263			"""
264			env = os.environ.copy()
265
266			if 'PYTHONPATH' in env:
267			del env['PYTHONPATH']
268
269			return env
270

StackStorm / st2

Pull Request — master (#3556)

install_requirement() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like