|
1
|
|
|
# -*- coding: utf-8 -*- |
|
2
|
|
|
from __future__ import absolute_import, division, print_function, unicode_literals |
|
3
|
|
|
|
|
4
|
|
|
from errno import EACCES, ELOOP, EPERM |
|
5
|
|
|
from io import open |
|
6
|
|
|
from logging import getLogger |
|
7
|
|
|
import os |
|
8
|
|
|
from os import X_OK, access, fstat |
|
9
|
|
|
from os.path import basename, dirname, isdir, isfile, join, splitext |
|
10
|
|
|
from shutil import copyfileobj, copystat |
|
11
|
|
|
import sys |
|
12
|
|
|
import tarfile |
|
13
|
|
|
|
|
14
|
|
|
from . import mkdir_p |
|
15
|
|
|
from .delete import rm_rf |
|
16
|
|
|
from .link import islink, lexists, link, readlink, symlink |
|
17
|
|
|
from .permissions import make_executable |
|
18
|
|
|
from .update import touch |
|
19
|
|
|
from ..subprocess import subprocess_call |
|
20
|
|
|
from ... import CondaError |
|
21
|
|
|
from ..._vendor.auxlib.ish import dals |
|
22
|
|
|
from ...base.constants import PACKAGE_CACHE_MAGIC_FILE |
|
23
|
|
|
from ...base.context import context |
|
24
|
|
|
from ...common.compat import ensure_binary, on_win |
|
25
|
|
|
from ...common.path import ensure_pad, expand, win_path_double_escape, win_path_ok |
|
26
|
|
|
from ...common.serialize import json_dump |
|
27
|
|
|
from ...exceptions import (BasicClobberError, CaseInsensitiveFileSystemError, CondaOSError, |
|
28
|
|
|
maybe_raise) |
|
29
|
|
|
from ...models.enums import FileMode, LinkType |
|
30
|
|
|
|
|
31
|
|
|
log = getLogger(__name__) |
|
32
|
|
|
stdoutlog = getLogger('conda.stdoutlog') |
|
33
|
|
|
|
|
34
|
|
|
# in __init__.py to help with circular imports |
|
35
|
|
|
mkdir_p = mkdir_p |
|
36
|
|
|
|
|
37
|
|
|
python_entry_point_template = dals(""" |
|
38
|
|
|
# -*- coding: utf-8 -*- |
|
39
|
|
|
import re |
|
40
|
|
|
import sys |
|
41
|
|
|
|
|
42
|
|
|
from %(module)s import %(import_name)s |
|
43
|
|
|
|
|
44
|
|
|
if __name__ == '__main__': |
|
45
|
|
|
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) |
|
46
|
|
|
sys.exit(%(func)s()) |
|
47
|
|
|
""") |
|
48
|
|
|
|
|
49
|
|
|
application_entry_point_template = dals(""" |
|
50
|
|
|
# -*- coding: utf-8 -*- |
|
51
|
|
|
if __name__ == '__main__': |
|
52
|
|
|
import os |
|
53
|
|
|
import sys |
|
54
|
|
|
args = ["%(source_full_path)s"] |
|
55
|
|
|
if len(sys.argv) > 1: |
|
56
|
|
|
args += sys.argv[1:] |
|
57
|
|
|
os.execv(args[0], args) |
|
58
|
|
|
""") |
|
59
|
|
|
|
|
60
|
|
|
|
|
61
|
|
|
def write_as_json_to_file(file_path, obj): |
|
62
|
|
|
log.trace("writing json to file %s", file_path) |
|
63
|
|
|
with open(file_path, str('wb')) as fo: |
|
64
|
|
|
json_str = json_dump(obj) |
|
65
|
|
|
fo.write(ensure_binary(json_str)) |
|
66
|
|
|
|
|
67
|
|
|
|
|
68
|
|
|
def create_python_entry_point(target_full_path, python_full_path, module, func): |
|
69
|
|
|
if lexists(target_full_path): |
|
70
|
|
|
maybe_raise(BasicClobberError( |
|
71
|
|
|
source_path=None, |
|
72
|
|
|
target_path=target_full_path, |
|
73
|
|
|
context=context, |
|
74
|
|
|
), context) |
|
75
|
|
|
|
|
76
|
|
|
import_name = func.split('.')[0] |
|
77
|
|
|
pyscript = python_entry_point_template % { |
|
78
|
|
|
'module': module, |
|
79
|
|
|
'func': func, |
|
80
|
|
|
'import_name': import_name, |
|
81
|
|
|
} |
|
82
|
|
|
|
|
83
|
|
|
if python_full_path is not None: |
|
84
|
|
|
shebang = '#!%s\n' % python_full_path |
|
85
|
|
|
if hasattr(shebang, 'encode'): |
|
86
|
|
|
shebang = shebang.encode() |
|
87
|
|
|
|
|
88
|
|
|
from ...core.portability import replace_long_shebang # TODO: must be in wrong spot |
|
89
|
|
|
shebang = replace_long_shebang(FileMode.text, shebang) |
|
90
|
|
|
|
|
91
|
|
|
if hasattr(shebang, 'decode'): |
|
92
|
|
|
shebang = shebang.decode() |
|
93
|
|
|
else: |
|
94
|
|
|
shebang = None |
|
95
|
|
|
|
|
96
|
|
|
with open(target_full_path, str('w')) as fo: |
|
97
|
|
|
if shebang is not None: |
|
98
|
|
|
fo.write(shebang) |
|
99
|
|
|
fo.write(pyscript) |
|
100
|
|
|
|
|
101
|
|
|
if shebang is not None: |
|
102
|
|
|
make_executable(target_full_path) |
|
103
|
|
|
|
|
104
|
|
|
return target_full_path |
|
105
|
|
|
|
|
106
|
|
|
|
|
107
|
|
|
def create_application_entry_point(source_full_path, target_full_path, python_full_path): |
|
108
|
|
|
# source_full_path: where the entry point file points to |
|
109
|
|
|
# target_full_path: the location of the new entry point file being created |
|
110
|
|
|
if lexists(target_full_path): |
|
111
|
|
|
maybe_raise(BasicClobberError( |
|
112
|
|
|
source_path=None, |
|
113
|
|
|
target_path=target_full_path, |
|
114
|
|
|
context=context, |
|
115
|
|
|
), context) |
|
116
|
|
|
|
|
117
|
|
|
entry_point = application_entry_point_template % { |
|
118
|
|
|
"source_full_path": win_path_double_escape(source_full_path), |
|
119
|
|
|
} |
|
120
|
|
|
if not isdir(dirname(target_full_path)): |
|
121
|
|
|
mkdir_p(dirname(target_full_path)) |
|
122
|
|
|
with open(target_full_path, str("w")) as fo: |
|
123
|
|
|
if ' ' in python_full_path: |
|
124
|
|
|
python_full_path = ensure_pad(python_full_path, '"') |
|
125
|
|
|
fo.write('#!%s\n' % python_full_path) |
|
126
|
|
|
fo.write(entry_point) |
|
127
|
|
|
make_executable(target_full_path) |
|
128
|
|
|
|
|
129
|
|
|
|
|
130
|
|
|
class ProgressFileWrapper(object): |
|
131
|
|
|
def __init__(self, fileobj, progress_update_callback): |
|
132
|
|
|
self.progress_file = fileobj |
|
133
|
|
|
self.progress_update_callback = progress_update_callback |
|
134
|
|
|
self.progress_file_size = max(1, fstat(fileobj.fileno()).st_size) |
|
135
|
|
|
|
|
136
|
|
|
def __getattr__(self, name): |
|
137
|
|
|
return getattr(self.progress_file, name) |
|
138
|
|
|
|
|
139
|
|
|
def __setattr__(self, name, value): |
|
140
|
|
|
if name.startswith("progress_"): |
|
141
|
|
|
super(ProgressFileWrapper, self).__setattr__(name, value) |
|
142
|
|
|
else: |
|
143
|
|
|
setattr(self.progress_file, name, value) |
|
144
|
|
|
|
|
145
|
|
|
def read(self, size=-1): |
|
146
|
|
|
data = self.progress_file.read(size) |
|
147
|
|
|
self.progress_update() |
|
148
|
|
|
return data |
|
149
|
|
|
|
|
150
|
|
|
def progress_update(self): |
|
151
|
|
|
rel_pos = self.progress_file.tell() / self.progress_file_size |
|
152
|
|
|
self.progress_update_callback(rel_pos) |
|
153
|
|
|
|
|
154
|
|
|
|
|
155
|
|
|
def extract_tarball(tarball_full_path, destination_directory=None, progress_update_callback=None): |
|
156
|
|
|
if destination_directory is None: |
|
157
|
|
|
destination_directory = tarball_full_path[:-8] |
|
158
|
|
|
log.debug("extracting %s\n to %s", tarball_full_path, destination_directory) |
|
159
|
|
|
|
|
160
|
|
|
assert not lexists(destination_directory), destination_directory |
|
161
|
|
|
|
|
162
|
|
|
with open(tarball_full_path, 'rb') as fileobj: |
|
163
|
|
|
if progress_update_callback: |
|
164
|
|
|
fileobj = ProgressFileWrapper(fileobj, progress_update_callback) |
|
165
|
|
|
with tarfile.open(fileobj=fileobj) as tar_file: |
|
166
|
|
|
try: |
|
167
|
|
|
tar_file.extractall(path=destination_directory) |
|
168
|
|
|
except EnvironmentError as e: |
|
169
|
|
|
if e.errno == ELOOP: |
|
170
|
|
|
raise CaseInsensitiveFileSystemError( |
|
171
|
|
|
package_location=tarball_full_path, |
|
172
|
|
|
extract_location=destination_directory, |
|
173
|
|
|
caused_by=e, |
|
174
|
|
|
) |
|
175
|
|
|
else: |
|
176
|
|
|
raise |
|
177
|
|
|
|
|
178
|
|
|
if sys.platform.startswith('linux') and os.getuid() == 0: |
|
179
|
|
|
# When extracting as root, tarfile will by restore ownership |
|
180
|
|
|
# of extracted files. However, we want root to be the owner |
|
181
|
|
|
# (our implementation of --no-same-owner). |
|
182
|
|
|
for root, dirs, files in os.walk(destination_directory): |
|
183
|
|
|
for fn in files: |
|
184
|
|
|
p = join(root, fn) |
|
185
|
|
|
os.lchown(p, 0, 0) |
|
186
|
|
|
|
|
187
|
|
|
|
|
188
|
|
|
def make_menu(prefix, file_path, remove=False): |
|
189
|
|
|
""" |
|
190
|
|
|
Create cross-platform menu items (e.g. Windows Start Menu) |
|
191
|
|
|
|
|
192
|
|
|
Passes all menu config files %PREFIX%/Menu/*.json to ``menuinst.install``. |
|
193
|
|
|
``remove=True`` will remove the menu items. |
|
194
|
|
|
""" |
|
195
|
|
|
if not on_win: |
|
196
|
|
|
return |
|
197
|
|
|
elif basename(prefix).startswith('_'): |
|
198
|
|
|
log.warn("Environment name starts with underscore '_'. Skipping menu installation.") |
|
199
|
|
|
return |
|
200
|
|
|
|
|
201
|
|
|
try: |
|
202
|
|
|
import menuinst |
|
203
|
|
|
menuinst.install(join(prefix, win_path_ok(file_path)), remove, prefix) |
|
204
|
|
|
except: |
|
205
|
|
|
stdoutlog.error("menuinst Exception", exc_info=True) |
|
206
|
|
|
|
|
207
|
|
|
|
|
208
|
|
|
def create_hard_link_or_copy(src, dst): |
|
209
|
|
|
if islink(src): |
|
210
|
|
|
message = dals(""" |
|
211
|
|
|
Cannot hard link a soft link |
|
212
|
|
|
source: %(source_path)s |
|
213
|
|
|
destination: %(destination_path)s |
|
214
|
|
|
""" % { |
|
215
|
|
|
'source_path': src, |
|
216
|
|
|
'destination_path': dst, |
|
217
|
|
|
}) |
|
218
|
|
|
raise CondaOSError(message) |
|
219
|
|
|
|
|
220
|
|
|
try: |
|
221
|
|
|
log.trace("creating hard link %s => %s", src, dst) |
|
222
|
|
|
link(src, dst) |
|
223
|
|
|
except (IOError, OSError): |
|
224
|
|
|
log.info('hard link failed, so copying %s => %s', src, dst) |
|
225
|
|
|
_do_copy(src, dst) |
|
226
|
|
|
|
|
227
|
|
|
|
|
228
|
|
|
def _is_unix_executable_using_ORIGIN(path): |
|
229
|
|
|
if on_win: |
|
230
|
|
|
return False |
|
231
|
|
|
else: |
|
232
|
|
|
return isfile(path) and not islink(path) and access(path, X_OK) |
|
233
|
|
|
|
|
234
|
|
|
|
|
235
|
|
|
def _do_softlink(src, dst): |
|
236
|
|
|
if _is_unix_executable_using_ORIGIN(src): |
|
237
|
|
|
# for extra details, see https://github.com/conda/conda/pull/4625#issuecomment-280696371 |
|
238
|
|
|
# We only need to do this copy for executables which have an RPATH containing $ORIGIN |
|
239
|
|
|
# on Linux, so `is_executable()` is currently overly aggressive. |
|
240
|
|
|
# A future optimization will be to copy code from @mingwandroid's virtualenv patch. |
|
241
|
|
|
copy(src, dst) |
|
242
|
|
|
else: |
|
243
|
|
|
log.trace("soft linking %s => %s", src, dst) |
|
244
|
|
|
symlink(src, dst) |
|
245
|
|
|
|
|
246
|
|
|
|
|
247
|
|
|
def create_fake_executable_softlink(src, dst): |
|
248
|
|
|
assert on_win |
|
249
|
|
|
src_root, _ = splitext(src) |
|
250
|
|
|
# TODO: this open will clobber, consider raising |
|
251
|
|
|
with open(dst, 'w') as f: |
|
252
|
|
|
f.write("@echo off\n" |
|
253
|
|
|
"call \"%s\" %%*\n" |
|
254
|
|
|
"" % src_root) |
|
255
|
|
|
return dst |
|
256
|
|
|
|
|
257
|
|
|
|
|
258
|
|
|
def copy(src, dst): |
|
259
|
|
|
# on unix, make sure relative symlinks stay symlinks |
|
260
|
|
|
if not on_win and islink(src): |
|
261
|
|
|
src_points_to = readlink(src) |
|
262
|
|
|
if not src_points_to.startswith('/'): |
|
263
|
|
|
# copy relative symlinks as symlinks |
|
264
|
|
|
log.trace("soft linking %s => %s", src, dst) |
|
265
|
|
|
symlink(src_points_to, dst) |
|
266
|
|
|
return |
|
267
|
|
|
_do_copy(src, dst) |
|
268
|
|
|
|
|
269
|
|
|
|
|
270
|
|
|
def _do_copy(src, dst): |
|
271
|
|
|
log.trace("copying %s => %s", src, dst) |
|
272
|
|
|
# src and dst are always files. So we can bypass some checks that shutil.copy does. |
|
273
|
|
|
# Also shutil.copy calls shutil.copymode, which we can skip because we are explicitly |
|
274
|
|
|
# calling copystat. |
|
275
|
|
|
|
|
276
|
|
|
# Same size as used by Linux cp command (has performance advantage). |
|
277
|
|
|
# Python's default is 16k. |
|
278
|
|
|
buffer_size = 4194304 # 4 * 1024 * 1024 == 4 MB |
|
279
|
|
|
with open(src, 'rb') as fsrc: |
|
280
|
|
|
with open(dst, 'wb') as fdst: |
|
281
|
|
|
copyfileobj(fsrc, fdst, buffer_size) |
|
282
|
|
|
|
|
283
|
|
|
try: |
|
284
|
|
|
copystat(src, dst) |
|
285
|
|
|
except (IOError, OSError) as e: # pragma: no cover |
|
286
|
|
|
# shutil.copystat gives a permission denied when using the os.setxattr function |
|
287
|
|
|
# on the security.selinux property. |
|
288
|
|
|
log.debug('%r', e) |
|
289
|
|
|
|
|
290
|
|
|
|
|
291
|
|
|
def create_link(src, dst, link_type=LinkType.hardlink, force=False): |
|
292
|
|
|
if link_type == LinkType.directory: |
|
293
|
|
|
# A directory is technically not a link. So link_type is a misnomer. |
|
294
|
|
|
# Naming is hard. |
|
295
|
|
|
if lexists(dst) and not isdir(dst): |
|
296
|
|
|
if not force: |
|
297
|
|
|
maybe_raise(BasicClobberError(src, dst, context), context) |
|
298
|
|
|
log.info("file exists, but clobbering for directory: %r" % dst) |
|
299
|
|
|
rm_rf(dst) |
|
300
|
|
|
mkdir_p(dst) |
|
301
|
|
|
return |
|
302
|
|
|
|
|
303
|
|
|
if not lexists(src): |
|
304
|
|
|
raise CondaError("Cannot link a source that does not exist. %s\n" |
|
305
|
|
|
"Running `conda clean --packages` may resolve your problem." % src) |
|
306
|
|
|
|
|
307
|
|
|
if lexists(dst): |
|
308
|
|
|
if not force: |
|
309
|
|
|
maybe_raise(BasicClobberError(src, dst, context), context) |
|
310
|
|
|
log.info("file exists, but clobbering: %r" % dst) |
|
311
|
|
|
rm_rf(dst) |
|
312
|
|
|
|
|
313
|
|
|
if link_type == LinkType.hardlink: |
|
314
|
|
|
if isdir(src): |
|
315
|
|
|
raise CondaError("Cannot hard link a directory. %s" % src) |
|
316
|
|
|
try: |
|
317
|
|
|
log.trace("hard linking %s => %s", src, dst) |
|
318
|
|
|
link(src, dst) |
|
319
|
|
|
except (IOError, OSError) as e: |
|
320
|
|
|
log.debug("%r", e) |
|
321
|
|
|
log.debug("hard-link failed. falling back to copy\n" |
|
322
|
|
|
" error: %r\n" |
|
323
|
|
|
" src: %s\n" |
|
324
|
|
|
" dst: %s", e, src, dst) |
|
325
|
|
|
copy(src, dst) |
|
326
|
|
|
elif link_type == LinkType.softlink: |
|
327
|
|
|
_do_softlink(src, dst) |
|
328
|
|
|
elif link_type == LinkType.copy: |
|
329
|
|
|
copy(src, dst) |
|
330
|
|
|
else: |
|
331
|
|
|
raise CondaError("Did not expect linktype=%r" % link_type) |
|
332
|
|
|
|
|
333
|
|
|
|
|
334
|
|
|
def compile_pyc(python_exe_full_path, py_full_path, pyc_full_path): |
|
335
|
|
|
if lexists(pyc_full_path): |
|
336
|
|
|
maybe_raise(BasicClobberError(None, pyc_full_path, context), context) |
|
337
|
|
|
|
|
338
|
|
|
command = '"%s" -Wi -m py_compile "%s"' % (python_exe_full_path, py_full_path) |
|
339
|
|
|
log.trace(command) |
|
340
|
|
|
result = subprocess_call(command, raise_on_error=False) |
|
341
|
|
|
|
|
342
|
|
|
if not isfile(pyc_full_path): |
|
343
|
|
|
message = dals(""" |
|
344
|
|
|
pyc file failed to compile successfully |
|
345
|
|
|
python_exe_full_path: %s |
|
346
|
|
|
py_full_path: %s |
|
347
|
|
|
pyc_full_path: %s |
|
348
|
|
|
compile rc: %s |
|
349
|
|
|
compile stdout: %s |
|
350
|
|
|
compile stderr: %s |
|
351
|
|
|
""") |
|
352
|
|
|
log.info(message, python_exe_full_path, py_full_path, pyc_full_path, |
|
353
|
|
|
result.rc, result.stdout, result.stderr) |
|
354
|
|
|
return None |
|
355
|
|
|
|
|
356
|
|
|
return pyc_full_path |
|
357
|
|
|
|
|
358
|
|
|
|
|
359
|
|
|
def create_package_cache_directory(pkgs_dir): |
|
360
|
|
|
# returns False if package cache directory cannot be created |
|
361
|
|
|
try: |
|
362
|
|
|
log.trace("creating package cache directory '%s'", pkgs_dir) |
|
363
|
|
|
sudo_safe = expand(pkgs_dir).startswith(expand('~')) |
|
364
|
|
|
touch(join(pkgs_dir, PACKAGE_CACHE_MAGIC_FILE), mkdir=True, sudo_safe=sudo_safe) |
|
365
|
|
|
touch(join(pkgs_dir, 'urls'), sudo_safe=sudo_safe) |
|
366
|
|
|
except (IOError, OSError) as e: |
|
367
|
|
|
if e.errno in (EACCES, EPERM): |
|
368
|
|
|
log.trace("cannot create package cache directory '%s'", pkgs_dir) |
|
369
|
|
|
return False |
|
370
|
|
|
else: |
|
371
|
|
|
raise |
|
372
|
|
|
return True |
|
373
|
|
|
|
|
374
|
|
|
|
|
375
|
|
|
def create_envs_directory(envs_dir): |
|
376
|
|
|
# returns False if envs directory cannot be created |
|
377
|
|
|
|
|
378
|
|
|
# The magic file being used here could change in the future. Don't write programs |
|
379
|
|
|
# outside this code base that rely on the presence of this file. |
|
380
|
|
|
# This value is duplicated in conda.base.context._first_writable_envs_dir(). |
|
381
|
|
|
envs_dir_magic_file = join(envs_dir, '.conda_envs_dir_test') |
|
382
|
|
|
try: |
|
383
|
|
|
log.trace("creating envs directory '%s'", envs_dir) |
|
384
|
|
|
sudo_safe = expand(envs_dir).startswith(expand('~')) |
|
385
|
|
|
touch(join(envs_dir, envs_dir_magic_file), mkdir=True, sudo_safe=sudo_safe) |
|
386
|
|
|
except (IOError, OSError) as e: |
|
387
|
|
|
if e.errno in (EACCES, EPERM): |
|
388
|
|
|
log.trace("cannot create envs directory '%s'", envs_dir) |
|
389
|
|
|
return False |
|
390
|
|
|
else: |
|
391
|
|
|
raise |
|
392
|
|
|
return True |
|
393
|
|
|
|
conda /
conda
