1
|
|
|
# -*- coding: utf-8 -*- |
2
|
|
|
from __future__ import absolute_import, division, print_function, unicode_literals |
3
|
|
|
|
4
|
|
|
from errno import EACCES, ELOOP, EPERM |
5
|
|
|
from io import open |
6
|
|
|
from logging import getLogger |
7
|
|
|
import os |
8
|
|
|
from os import X_OK, access, fstat |
9
|
|
|
from os.path import basename, dirname, isdir, isfile, join, splitext |
10
|
|
|
from shutil import copyfileobj, copystat |
11
|
|
|
import sys |
12
|
|
|
import tarfile |
13
|
|
|
|
14
|
|
|
from . import mkdir_p |
15
|
|
|
from .delete import rm_rf |
16
|
|
|
from .link import islink, lexists, link, readlink, symlink |
17
|
|
|
from .permissions import make_executable |
18
|
|
|
from .update import touch |
19
|
|
|
from ..subprocess import subprocess_call |
20
|
|
|
from ... import CondaError |
21
|
|
|
from ..._vendor.auxlib.ish import dals |
22
|
|
|
from ...base.constants import PACKAGE_CACHE_MAGIC_FILE |
23
|
|
|
from ...base.context import context |
24
|
|
|
from ...common.compat import ensure_binary, on_win |
25
|
|
|
from ...common.path import ensure_pad, expand, win_path_double_escape, win_path_ok |
26
|
|
|
from ...common.serialize import json_dump |
27
|
|
|
from ...exceptions import (BasicClobberError, CaseInsensitiveFileSystemError, CondaOSError, |
28
|
|
|
maybe_raise) |
29
|
|
|
from ...models.enums import FileMode, LinkType |
30
|
|
|
|
31
|
|
|
log = getLogger(__name__) |
32
|
|
|
stdoutlog = getLogger('conda.stdoutlog') |
33
|
|
|
|
34
|
|
|
# in __init__.py to help with circular imports |
35
|
|
|
mkdir_p = mkdir_p |
36
|
|
|
|
37
|
|
|
python_entry_point_template = dals(""" |
38
|
|
|
# -*- coding: utf-8 -*- |
39
|
|
|
import re |
40
|
|
|
import sys |
41
|
|
|
|
42
|
|
|
from %(module)s import %(import_name)s |
43
|
|
|
|
44
|
|
|
if __name__ == '__main__': |
45
|
|
|
sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) |
46
|
|
|
sys.exit(%(func)s()) |
47
|
|
|
""") |
48
|
|
|
|
49
|
|
|
application_entry_point_template = dals(""" |
50
|
|
|
# -*- coding: utf-8 -*- |
51
|
|
|
if __name__ == '__main__': |
52
|
|
|
import os |
53
|
|
|
import sys |
54
|
|
|
args = ["%(source_full_path)s"] |
55
|
|
|
if len(sys.argv) > 1: |
56
|
|
|
args += sys.argv[1:] |
57
|
|
|
os.execv(args[0], args) |
58
|
|
|
""") |
59
|
|
|
|
60
|
|
|
|
61
|
|
|
def write_as_json_to_file(file_path, obj): |
62
|
|
|
log.trace("writing json to file %s", file_path) |
63
|
|
|
with open(file_path, str('wb')) as fo: |
64
|
|
|
json_str = json_dump(obj) |
65
|
|
|
fo.write(ensure_binary(json_str)) |
66
|
|
|
|
67
|
|
|
|
68
|
|
|
def create_python_entry_point(target_full_path, python_full_path, module, func): |
69
|
|
|
if lexists(target_full_path): |
70
|
|
|
maybe_raise(BasicClobberError( |
71
|
|
|
source_path=None, |
72
|
|
|
target_path=target_full_path, |
73
|
|
|
context=context, |
74
|
|
|
), context) |
75
|
|
|
|
76
|
|
|
import_name = func.split('.')[0] |
77
|
|
|
pyscript = python_entry_point_template % { |
78
|
|
|
'module': module, |
79
|
|
|
'func': func, |
80
|
|
|
'import_name': import_name, |
81
|
|
|
} |
82
|
|
|
|
83
|
|
|
if python_full_path is not None: |
84
|
|
|
shebang = '#!%s\n' % python_full_path |
85
|
|
|
if hasattr(shebang, 'encode'): |
86
|
|
|
shebang = shebang.encode() |
87
|
|
|
|
88
|
|
|
from ...core.portability import replace_long_shebang # TODO: must be in wrong spot |
89
|
|
|
shebang = replace_long_shebang(FileMode.text, shebang) |
90
|
|
|
|
91
|
|
|
if hasattr(shebang, 'decode'): |
92
|
|
|
shebang = shebang.decode() |
93
|
|
|
else: |
94
|
|
|
shebang = None |
95
|
|
|
|
96
|
|
|
with open(target_full_path, str('w')) as fo: |
97
|
|
|
if shebang is not None: |
98
|
|
|
fo.write(shebang) |
99
|
|
|
fo.write(pyscript) |
100
|
|
|
|
101
|
|
|
if shebang is not None: |
102
|
|
|
make_executable(target_full_path) |
103
|
|
|
|
104
|
|
|
return target_full_path |
105
|
|
|
|
106
|
|
|
|
107
|
|
|
def create_application_entry_point(source_full_path, target_full_path, python_full_path): |
108
|
|
|
# source_full_path: where the entry point file points to |
109
|
|
|
# target_full_path: the location of the new entry point file being created |
110
|
|
|
if lexists(target_full_path): |
111
|
|
|
maybe_raise(BasicClobberError( |
112
|
|
|
source_path=None, |
113
|
|
|
target_path=target_full_path, |
114
|
|
|
context=context, |
115
|
|
|
), context) |
116
|
|
|
|
117
|
|
|
entry_point = application_entry_point_template % { |
118
|
|
|
"source_full_path": win_path_double_escape(source_full_path), |
119
|
|
|
} |
120
|
|
|
if not isdir(dirname(target_full_path)): |
121
|
|
|
mkdir_p(dirname(target_full_path)) |
122
|
|
|
with open(target_full_path, str("w")) as fo: |
123
|
|
|
if ' ' in python_full_path: |
124
|
|
|
python_full_path = ensure_pad(python_full_path, '"') |
125
|
|
|
fo.write('#!%s\n' % python_full_path) |
126
|
|
|
fo.write(entry_point) |
127
|
|
|
make_executable(target_full_path) |
128
|
|
|
|
129
|
|
|
|
130
|
|
|
class ProgressFileWrapper(object): |
131
|
|
|
def __init__(self, fileobj, progress_update_callback): |
132
|
|
|
self.progress_file = fileobj |
133
|
|
|
self.progress_update_callback = progress_update_callback |
134
|
|
|
self.progress_file_size = max(1, fstat(fileobj.fileno()).st_size) |
135
|
|
|
|
136
|
|
|
def __getattr__(self, name): |
137
|
|
|
return getattr(self.progress_file, name) |
138
|
|
|
|
139
|
|
|
def __setattr__(self, name, value): |
140
|
|
|
if name.startswith("progress_"): |
141
|
|
|
super(ProgressFileWrapper, self).__setattr__(name, value) |
142
|
|
|
else: |
143
|
|
|
setattr(self.progress_file, name, value) |
144
|
|
|
|
145
|
|
|
def read(self, size=-1): |
146
|
|
|
data = self.progress_file.read(size) |
147
|
|
|
self.progress_update() |
148
|
|
|
return data |
149
|
|
|
|
150
|
|
|
def progress_update(self): |
151
|
|
|
rel_pos = self.progress_file.tell() / self.progress_file_size |
152
|
|
|
self.progress_update_callback(rel_pos) |
153
|
|
|
|
154
|
|
|
|
155
|
|
|
def extract_tarball(tarball_full_path, destination_directory=None, progress_update_callback=None): |
156
|
|
|
if destination_directory is None: |
157
|
|
|
destination_directory = tarball_full_path[:-8] |
158
|
|
|
log.debug("extracting %s\n to %s", tarball_full_path, destination_directory) |
159
|
|
|
|
160
|
|
|
assert not lexists(destination_directory), destination_directory |
161
|
|
|
|
162
|
|
|
with open(tarball_full_path, 'rb') as fileobj: |
163
|
|
|
if progress_update_callback: |
164
|
|
|
fileobj = ProgressFileWrapper(fileobj, progress_update_callback) |
165
|
|
|
with tarfile.open(fileobj=fileobj) as tar_file: |
166
|
|
|
try: |
167
|
|
|
tar_file.extractall(path=destination_directory) |
168
|
|
|
except EnvironmentError as e: |
169
|
|
|
if e.errno == ELOOP: |
170
|
|
|
raise CaseInsensitiveFileSystemError( |
171
|
|
|
package_location=tarball_full_path, |
172
|
|
|
extract_location=destination_directory, |
173
|
|
|
caused_by=e, |
174
|
|
|
) |
175
|
|
|
else: |
176
|
|
|
raise |
177
|
|
|
|
178
|
|
|
if sys.platform.startswith('linux') and os.getuid() == 0: |
179
|
|
|
# When extracting as root, tarfile will by restore ownership |
180
|
|
|
# of extracted files. However, we want root to be the owner |
181
|
|
|
# (our implementation of --no-same-owner). |
182
|
|
|
for root, dirs, files in os.walk(destination_directory): |
183
|
|
|
for fn in files: |
184
|
|
|
p = join(root, fn) |
185
|
|
|
os.lchown(p, 0, 0) |
186
|
|
|
|
187
|
|
|
|
188
|
|
|
def make_menu(prefix, file_path, remove=False): |
189
|
|
|
""" |
190
|
|
|
Create cross-platform menu items (e.g. Windows Start Menu) |
191
|
|
|
|
192
|
|
|
Passes all menu config files %PREFIX%/Menu/*.json to ``menuinst.install``. |
193
|
|
|
``remove=True`` will remove the menu items. |
194
|
|
|
""" |
195
|
|
|
if not on_win: |
196
|
|
|
return |
197
|
|
|
elif basename(prefix).startswith('_'): |
198
|
|
|
log.warn("Environment name starts with underscore '_'. Skipping menu installation.") |
199
|
|
|
return |
200
|
|
|
|
201
|
|
|
try: |
202
|
|
|
import menuinst |
203
|
|
|
menuinst.install(join(prefix, win_path_ok(file_path)), remove, prefix) |
204
|
|
|
except: |
205
|
|
|
stdoutlog.error("menuinst Exception", exc_info=True) |
206
|
|
|
|
207
|
|
|
|
208
|
|
|
def create_hard_link_or_copy(src, dst): |
209
|
|
|
if islink(src): |
210
|
|
|
message = dals(""" |
211
|
|
|
Cannot hard link a soft link |
212
|
|
|
source: %(source_path)s |
213
|
|
|
destination: %(destination_path)s |
214
|
|
|
""" % { |
215
|
|
|
'source_path': src, |
216
|
|
|
'destination_path': dst, |
217
|
|
|
}) |
218
|
|
|
raise CondaOSError(message) |
219
|
|
|
|
220
|
|
|
try: |
221
|
|
|
log.trace("creating hard link %s => %s", src, dst) |
222
|
|
|
link(src, dst) |
223
|
|
|
except (IOError, OSError): |
224
|
|
|
log.info('hard link failed, so copying %s => %s', src, dst) |
225
|
|
|
_do_copy(src, dst) |
226
|
|
|
|
227
|
|
|
|
228
|
|
|
def _is_unix_executable_using_ORIGIN(path): |
229
|
|
|
if on_win: |
230
|
|
|
return False |
231
|
|
|
else: |
232
|
|
|
return isfile(path) and not islink(path) and access(path, X_OK) |
233
|
|
|
|
234
|
|
|
|
235
|
|
|
def _do_softlink(src, dst): |
236
|
|
|
if _is_unix_executable_using_ORIGIN(src): |
237
|
|
|
# for extra details, see https://github.com/conda/conda/pull/4625#issuecomment-280696371 |
238
|
|
|
# We only need to do this copy for executables which have an RPATH containing $ORIGIN |
239
|
|
|
# on Linux, so `is_executable()` is currently overly aggressive. |
240
|
|
|
# A future optimization will be to copy code from @mingwandroid's virtualenv patch. |
241
|
|
|
copy(src, dst) |
242
|
|
|
else: |
243
|
|
|
log.trace("soft linking %s => %s", src, dst) |
244
|
|
|
symlink(src, dst) |
245
|
|
|
|
246
|
|
|
|
247
|
|
|
def create_fake_executable_softlink(src, dst): |
248
|
|
|
assert on_win |
249
|
|
|
src_root, _ = splitext(src) |
250
|
|
|
# TODO: this open will clobber, consider raising |
251
|
|
|
with open(dst, 'w') as f: |
252
|
|
|
f.write("@echo off\n" |
253
|
|
|
"call \"%s\" %%*\n" |
254
|
|
|
"" % src_root) |
255
|
|
|
return dst |
256
|
|
|
|
257
|
|
|
|
258
|
|
|
def copy(src, dst): |
259
|
|
|
# on unix, make sure relative symlinks stay symlinks |
260
|
|
|
if not on_win and islink(src): |
261
|
|
|
src_points_to = readlink(src) |
262
|
|
|
if not src_points_to.startswith('/'): |
263
|
|
|
# copy relative symlinks as symlinks |
264
|
|
|
log.trace("soft linking %s => %s", src, dst) |
265
|
|
|
symlink(src_points_to, dst) |
266
|
|
|
return |
267
|
|
|
_do_copy(src, dst) |
268
|
|
|
|
269
|
|
|
|
270
|
|
|
def _do_copy(src, dst): |
271
|
|
|
log.trace("copying %s => %s", src, dst) |
272
|
|
|
# src and dst are always files. So we can bypass some checks that shutil.copy does. |
273
|
|
|
# Also shutil.copy calls shutil.copymode, which we can skip because we are explicitly |
274
|
|
|
# calling copystat. |
275
|
|
|
|
276
|
|
|
# Same size as used by Linux cp command (has performance advantage). |
277
|
|
|
# Python's default is 16k. |
278
|
|
|
buffer_size = 4194304 # 4 * 1024 * 1024 == 4 MB |
279
|
|
|
with open(src, 'rb') as fsrc: |
280
|
|
|
with open(dst, 'wb') as fdst: |
281
|
|
|
copyfileobj(fsrc, fdst, buffer_size) |
282
|
|
|
|
283
|
|
|
try: |
284
|
|
|
copystat(src, dst) |
285
|
|
|
except (IOError, OSError) as e: # pragma: no cover |
286
|
|
|
# shutil.copystat gives a permission denied when using the os.setxattr function |
287
|
|
|
# on the security.selinux property. |
288
|
|
|
log.debug('%r', e) |
289
|
|
|
|
290
|
|
|
|
291
|
|
|
def create_link(src, dst, link_type=LinkType.hardlink, force=False): |
292
|
|
|
if link_type == LinkType.directory: |
293
|
|
|
# A directory is technically not a link. So link_type is a misnomer. |
294
|
|
|
# Naming is hard. |
295
|
|
|
if lexists(dst) and not isdir(dst): |
296
|
|
|
if not force: |
297
|
|
|
maybe_raise(BasicClobberError(src, dst, context), context) |
298
|
|
|
log.info("file exists, but clobbering for directory: %r" % dst) |
299
|
|
|
rm_rf(dst) |
300
|
|
|
mkdir_p(dst) |
301
|
|
|
return |
302
|
|
|
|
303
|
|
|
if not lexists(src): |
304
|
|
|
raise CondaError("Cannot link a source that does not exist. %s\n" |
305
|
|
|
"Running `conda clean --packages` may resolve your problem." % src) |
306
|
|
|
|
307
|
|
|
if lexists(dst): |
308
|
|
|
if not force: |
309
|
|
|
maybe_raise(BasicClobberError(src, dst, context), context) |
310
|
|
|
log.info("file exists, but clobbering: %r" % dst) |
311
|
|
|
rm_rf(dst) |
312
|
|
|
|
313
|
|
|
if link_type == LinkType.hardlink: |
314
|
|
|
if isdir(src): |
315
|
|
|
raise CondaError("Cannot hard link a directory. %s" % src) |
316
|
|
|
try: |
317
|
|
|
log.trace("hard linking %s => %s", src, dst) |
318
|
|
|
link(src, dst) |
319
|
|
|
except (IOError, OSError) as e: |
320
|
|
|
log.debug("%r", e) |
321
|
|
|
log.debug("hard-link failed. falling back to copy\n" |
322
|
|
|
" error: %r\n" |
323
|
|
|
" src: %s\n" |
324
|
|
|
" dst: %s", e, src, dst) |
325
|
|
|
copy(src, dst) |
326
|
|
|
elif link_type == LinkType.softlink: |
327
|
|
|
_do_softlink(src, dst) |
328
|
|
|
elif link_type == LinkType.copy: |
329
|
|
|
copy(src, dst) |
330
|
|
|
else: |
331
|
|
|
raise CondaError("Did not expect linktype=%r" % link_type) |
332
|
|
|
|
333
|
|
|
|
334
|
|
|
def compile_pyc(python_exe_full_path, py_full_path, pyc_full_path): |
335
|
|
|
if lexists(pyc_full_path): |
336
|
|
|
maybe_raise(BasicClobberError(None, pyc_full_path, context), context) |
337
|
|
|
|
338
|
|
|
command = '"%s" -Wi -m py_compile "%s"' % (python_exe_full_path, py_full_path) |
339
|
|
|
log.trace(command) |
340
|
|
|
result = subprocess_call(command, raise_on_error=False) |
341
|
|
|
|
342
|
|
|
if not isfile(pyc_full_path): |
343
|
|
|
message = dals(""" |
344
|
|
|
pyc file failed to compile successfully |
345
|
|
|
python_exe_full_path: %s |
346
|
|
|
py_full_path: %s |
347
|
|
|
pyc_full_path: %s |
348
|
|
|
compile rc: %s |
349
|
|
|
compile stdout: %s |
350
|
|
|
compile stderr: %s |
351
|
|
|
""") |
352
|
|
|
log.info(message, python_exe_full_path, py_full_path, pyc_full_path, |
353
|
|
|
result.rc, result.stdout, result.stderr) |
354
|
|
|
return None |
355
|
|
|
|
356
|
|
|
return pyc_full_path |
357
|
|
|
|
358
|
|
|
|
359
|
|
|
def create_package_cache_directory(pkgs_dir): |
360
|
|
|
# returns False if package cache directory cannot be created |
361
|
|
|
try: |
362
|
|
|
log.trace("creating package cache directory '%s'", pkgs_dir) |
363
|
|
|
sudo_safe = expand(pkgs_dir).startswith(expand('~')) |
364
|
|
|
touch(join(pkgs_dir, PACKAGE_CACHE_MAGIC_FILE), mkdir=True, sudo_safe=sudo_safe) |
365
|
|
|
touch(join(pkgs_dir, 'urls'), sudo_safe=sudo_safe) |
366
|
|
|
except (IOError, OSError) as e: |
367
|
|
|
if e.errno in (EACCES, EPERM): |
368
|
|
|
log.trace("cannot create package cache directory '%s'", pkgs_dir) |
369
|
|
|
return False |
370
|
|
|
else: |
371
|
|
|
raise |
372
|
|
|
return True |
373
|
|
|
|
374
|
|
|
|
375
|
|
|
def create_envs_directory(envs_dir): |
376
|
|
|
# returns False if envs directory cannot be created |
377
|
|
|
|
378
|
|
|
# The magic file being used here could change in the future. Don't write programs |
379
|
|
|
# outside this code base that rely on the presence of this file. |
380
|
|
|
# This value is duplicated in conda.base.context._first_writable_envs_dir(). |
381
|
|
|
envs_dir_magic_file = join(envs_dir, '.conda_envs_dir_test') |
382
|
|
|
try: |
383
|
|
|
log.trace("creating envs directory '%s'", envs_dir) |
384
|
|
|
sudo_safe = expand(envs_dir).startswith(expand('~')) |
385
|
|
|
touch(join(envs_dir, envs_dir_magic_file), mkdir=True, sudo_safe=sudo_safe) |
386
|
|
|
except (IOError, OSError) as e: |
387
|
|
|
if e.errno in (EACCES, EPERM): |
388
|
|
|
log.trace("cannot create envs directory '%s'", envs_dir) |
389
|
|
|
return False |
390
|
|
|
else: |
391
|
|
|
raise |
392
|
|
|
return True |
393
|
|
|
|