1
|
|
|
''' |
2
|
|
|
|
3
|
|
|
anaconda upload CONDA_PACKAGE_1.bz2 |
4
|
|
|
anaconda upload notebook.ipynb |
5
|
|
|
anaconda upload environment.yml |
6
|
|
|
|
7
|
|
|
##### See Also |
8
|
|
|
|
9
|
|
|
* [Uploading a Conda Package](http://docs.anaconda.org/using.html#Uploading) |
10
|
|
|
* [Uploading a PyPI Package](http://docs.anaconda.org/using.html#UploadingPypiPackages) |
11
|
|
|
|
12
|
|
|
''' |
13
|
|
|
from __future__ import unicode_literals |
14
|
|
|
|
15
|
|
|
import tempfile |
16
|
|
|
import argparse |
17
|
|
|
import subprocess |
18
|
|
|
from glob import glob |
19
|
|
|
import logging |
20
|
|
|
import os |
21
|
|
|
from os.path import exists |
22
|
|
|
import sys |
23
|
|
|
from collections import defaultdict |
24
|
|
|
|
25
|
|
|
import nbformat |
26
|
|
|
|
27
|
|
|
from binstar_client import errors |
28
|
|
|
from binstar_client.utils import bool_input |
29
|
|
|
from binstar_client.utils import get_server_api |
30
|
|
|
from binstar_client.utils import get_config |
31
|
|
|
from binstar_client.utils import upload_print_callback |
32
|
|
|
from binstar_client.utils.projects import upload_project |
33
|
|
|
from binstar_client.utils.detect import detect_package_type, get_attrs |
34
|
|
|
|
35
|
|
|
|
36
|
|
|
# Python 3 Support |
37
|
|
|
try: |
38
|
|
|
input = raw_input |
39
|
|
|
except NameError: |
40
|
|
|
input = input |
41
|
|
|
|
42
|
|
|
|
43
|
|
|
logger = logging.getLogger('binstar.upload') |
44
|
|
|
|
45
|
|
|
|
46
|
|
|
PACKAGE_TYPES = defaultdict(lambda: 'Package', {'env': 'Environment', 'ipynb': 'Notebook'}) |
47
|
|
|
|
48
|
|
|
|
49
|
|
|
def verbose_package_type(pkg_type, lowercase=True): |
50
|
|
|
verbose_type = PACKAGE_TYPES[pkg_type] |
51
|
|
|
if lowercase: |
52
|
|
|
verbose_type = verbose_type.lower() |
53
|
|
|
return verbose_type |
54
|
|
|
|
55
|
|
|
|
56
|
|
|
def create_release(aserver_api, username, package_name, version, release_attrs, announce=None): |
57
|
|
|
aserver_api.add_release(username, package_name, version, [], announce, release_attrs) |
58
|
|
|
|
59
|
|
|
|
60
|
|
|
def create_release_interactive(aserver_api, username, package_name, version, release_attrs): |
61
|
|
|
logger.info('\nThe release %s/%s/%s does not exist' % (username, package_name, version)) |
62
|
|
|
if not bool_input('Would you like to create it now?'): |
63
|
|
|
logger.info('good-bye') |
64
|
|
|
raise SystemExit(-1) |
65
|
|
|
|
66
|
|
|
description = input('Enter a short description of the release:\n') |
67
|
|
|
logger.info("\nAnnouncements are emailed to your package followers.") |
68
|
|
|
make_announcement = bool_input('Would you like to make an announcement to the package followers?', False) |
69
|
|
|
if make_announcement: |
70
|
|
|
announce = input('Markdown Announcement:\n') |
71
|
|
|
else: |
72
|
|
|
announce = '' |
73
|
|
|
|
74
|
|
|
aserver_api.add_release(username, package_name, version, [], announce, release_attrs) |
75
|
|
|
|
76
|
|
|
|
77
|
|
|
def determine_package_type(filename, args): |
78
|
|
|
""" |
79
|
|
|
return the file type from the inspected package or from the |
80
|
|
|
-t/--package-type argument |
81
|
|
|
""" |
82
|
|
|
if args.package_type: |
83
|
|
|
package_type = args.package_type |
84
|
|
|
else: |
85
|
|
|
logger.info('detecting file type ...') |
86
|
|
|
sys.stdout.flush() |
87
|
|
|
package_type = detect_package_type(filename) |
88
|
|
|
if package_type is None: |
89
|
|
|
message = 'Could not detect package type of file %r please specify package type with option --package-type' % filename |
90
|
|
|
logger.error(message) |
91
|
|
|
raise errors.BinstarError(message) |
92
|
|
|
logger.info(package_type) |
93
|
|
|
|
94
|
|
|
return package_type |
95
|
|
|
|
96
|
|
|
|
97
|
|
|
def get_package_name(args, package_attrs, filename, package_type): |
98
|
|
|
if args.package: |
99
|
|
|
if 'name' in package_attrs and package_attrs['name'].lower() != args.package.lower(): |
100
|
|
|
msg = 'Package name on the command line " {}" does not match the package name in the file "{}"'.format( |
101
|
|
|
args.package.lower(), package_attrs['name'].lower() |
102
|
|
|
) |
103
|
|
|
logger.error(msg) |
104
|
|
|
raise errors.BinstarError(msg) |
105
|
|
|
package_name = args.package |
106
|
|
|
else: |
107
|
|
|
if 'name' not in package_attrs: |
108
|
|
|
message = "Could not detect package name for package type %s, please use the --package option" % (package_type,) |
109
|
|
|
logger.error(message) |
110
|
|
|
raise errors.BinstarError(message) |
111
|
|
|
package_name = package_attrs['name'] |
112
|
|
|
|
113
|
|
|
return package_name |
114
|
|
|
|
115
|
|
|
|
116
|
|
|
def get_version(args, release_attrs, package_type): |
117
|
|
|
if args.version: |
118
|
|
|
version = args.version |
119
|
|
|
else: |
120
|
|
|
if 'version' not in release_attrs: |
121
|
|
|
message = "Could not detect package version for package type %s, please use the --version option" % (package_type,) |
122
|
|
|
logger.error(message) |
123
|
|
|
raise errors.BinstarError(message) |
124
|
|
|
version = release_attrs['version'] |
125
|
|
|
return version |
126
|
|
|
|
127
|
|
|
|
128
|
|
|
def add_package(aserver_api, args, username, package_name, package_attrs, package_type): |
129
|
|
|
try: |
130
|
|
|
return aserver_api.package(username, package_name) |
131
|
|
|
except errors.NotFound: |
132
|
|
|
if not args.auto_register: |
133
|
|
|
message = ( |
134
|
|
|
'Anaconda Cloud package %s/%s does not exist. ' |
135
|
|
|
'Please run "anaconda package --create" to create this package namespace in the cloud.' % |
136
|
|
|
(username, package_name) |
137
|
|
|
) |
138
|
|
|
logger.error(message) |
139
|
|
|
raise errors.UserError(message) |
140
|
|
|
else: |
141
|
|
|
|
142
|
|
|
if args.summary: |
143
|
|
|
summary = args.summary |
144
|
|
|
else: |
145
|
|
|
if 'summary' not in package_attrs: |
146
|
|
|
message = "Could not detect package summary for package type %s, please use the --summary option" % (package_type,) |
147
|
|
|
logger.error(message) |
148
|
|
|
raise errors.BinstarError(message) |
149
|
|
|
summary = package_attrs['summary'] |
150
|
|
|
|
151
|
|
|
public = not args.private |
152
|
|
|
|
153
|
|
|
return aserver_api.add_package( |
154
|
|
|
username, |
155
|
|
|
package_name, |
156
|
|
|
summary, |
157
|
|
|
package_attrs.get('license'), |
158
|
|
|
public=public, |
159
|
|
|
attrs=package_attrs, |
160
|
|
|
license_url=package_attrs.get('license_url'), |
161
|
|
|
license_family=package_attrs.get('license_family'), |
162
|
|
|
package_type=package_type, |
163
|
|
|
) |
164
|
|
|
|
165
|
|
|
|
166
|
|
|
def add_release(aserver_api, args, username, package_name, version, release_attrs): |
167
|
|
|
try: |
168
|
|
|
# Check if the release already exists |
169
|
|
|
aserver_api.release(username, package_name, version) |
170
|
|
|
except errors.NotFound: |
171
|
|
|
if args.mode == 'interactive': |
172
|
|
|
create_release_interactive(aserver_api, username, package_name, version, release_attrs) |
173
|
|
|
else: |
174
|
|
|
create_release(aserver_api, username, package_name, version, release_attrs) |
175
|
|
|
|
176
|
|
|
|
177
|
|
|
def remove_existing_file(aserver_api, args, username, package_name, version, file_attrs): |
178
|
|
|
try: |
179
|
|
|
aserver_api.distribution(username, package_name, version, file_attrs['basename']) |
180
|
|
|
except errors.NotFound: |
181
|
|
|
return False |
182
|
|
|
else: |
183
|
|
|
if args.mode == 'force': |
184
|
|
|
logger.warning('Distribution %s already exists ... removing' % (file_attrs['basename'],)) |
185
|
|
|
aserver_api.remove_dist(username, package_name, version, file_attrs['basename']) |
186
|
|
|
if args.mode == 'interactive': |
187
|
|
|
if bool_input('Distribution %s already exists. Would you like to replace it?' % (file_attrs['basename'],)): |
188
|
|
|
aserver_api.remove_dist(username, package_name, version, file_attrs['basename']) |
189
|
|
|
else: |
190
|
|
|
logger.info('Not replacing distribution %s' % (file_attrs['basename'],)) |
191
|
|
|
return True |
192
|
|
|
|
193
|
|
|
|
194
|
|
|
def upload_package(filename, package_type, aserver_api, username, args): |
195
|
|
|
logger.info('extracting {} attributes for upload ...'.format(verbose_package_type(package_type))) |
196
|
|
|
sys.stdout.flush() |
197
|
|
|
try: |
198
|
|
|
package_attrs, release_attrs, file_attrs = get_attrs(package_type, filename, parser_args=args) |
199
|
|
|
except Exception: |
200
|
|
|
message = 'Trouble reading metadata from {}. Is this a valid {} package?'.format( |
201
|
|
|
filename, verbose_package_type(package_type) |
202
|
|
|
) |
203
|
|
|
logger.error(message) |
204
|
|
|
if args.show_traceback: |
205
|
|
|
raise |
206
|
|
|
raise errors.BinstarError(message) |
207
|
|
|
|
208
|
|
|
if args.build_id: |
209
|
|
|
file_attrs['attrs']['binstar_build'] = args.build_id |
210
|
|
|
|
211
|
|
|
logger.info('done') |
212
|
|
|
|
213
|
|
|
package_name = get_package_name(args, package_attrs, filename, package_type) |
214
|
|
|
version = get_version(args, release_attrs, package_type) |
215
|
|
|
|
216
|
|
|
package = add_package(aserver_api, args, username, package_name, package_attrs, package_type) |
217
|
|
|
if package_type not in package.get('package_types', []): |
218
|
|
|
package_types = package.get('package_types') |
219
|
|
|
message = 'You already have a {} named \'{}\'. Use a different name for this {}.'.format( |
220
|
|
|
verbose_package_type(package_types[0] if package_types else ''), |
221
|
|
|
package_name, |
222
|
|
|
verbose_package_type(package_type), |
223
|
|
|
) |
224
|
|
|
logger.error(message) |
225
|
|
|
raise errors.BinstarError(message) |
226
|
|
|
add_release(aserver_api, args, username, package_name, version, release_attrs) |
227
|
|
|
binstar_package_type = file_attrs.pop('binstar_package_type', package_type) |
228
|
|
|
|
229
|
|
|
with open(filename, 'rb') as fd: |
230
|
|
|
logger.info('\nUploading file %s/%s/%s/%s ... ' % (username, package_name, version, file_attrs['basename'])) |
231
|
|
|
sys.stdout.flush() |
232
|
|
|
|
233
|
|
|
if remove_existing_file(aserver_api, args, username, package_name, version, file_attrs): |
234
|
|
|
return None |
235
|
|
|
try: |
236
|
|
|
upload_info = aserver_api.upload(username, |
237
|
|
|
package_name, |
238
|
|
|
version, |
239
|
|
|
file_attrs['basename'], |
240
|
|
|
fd, binstar_package_type, |
241
|
|
|
args.description, |
242
|
|
|
dependencies=file_attrs.get('dependencies'), |
243
|
|
|
attrs=file_attrs['attrs'], |
244
|
|
|
channels=args.labels, |
245
|
|
|
callback=upload_print_callback(args)) |
246
|
|
|
except errors.Conflict: |
247
|
|
|
full_name = '%s/%s/%s/%s' % (username, package_name, version, file_attrs['basename']) |
248
|
|
|
logger.info('Distribution already exists. Please use the ' |
249
|
|
|
'-i/--interactive or --force options or `anaconda remove %s`' % full_name) |
250
|
|
|
raise |
251
|
|
|
|
252
|
|
|
logger.info("\n\nUpload(s) Complete\n") |
253
|
|
|
return [package_name, upload_info] |
254
|
|
|
|
255
|
|
|
|
256
|
|
|
def get_convert_files(files): |
257
|
|
|
tmpdir = tempfile.mkdtemp() |
258
|
|
|
|
259
|
|
|
for filepath in files: |
260
|
|
|
logger.info('Running conda convert on %s', filepath) |
261
|
|
|
process = subprocess.Popen( |
262
|
|
|
['conda-convert', '-p', 'all', filepath, '-o', tmpdir], |
263
|
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE |
264
|
|
|
) |
265
|
|
|
stdout, stderr = process.communicate() |
266
|
|
|
|
267
|
|
|
if stderr: |
268
|
|
|
logger.warning('Couldn\'t generate platform packages for %s: %s', filepath, stderr) |
269
|
|
|
|
270
|
|
|
result = [] |
271
|
|
|
for path, dirs, files in os.walk(tmpdir): |
272
|
|
|
for filename in files: |
273
|
|
|
result.append(os.path.join(path, filename)) |
274
|
|
|
|
275
|
|
|
return result |
276
|
|
|
|
277
|
|
|
|
278
|
|
|
def main(args): |
279
|
|
|
aserver_api = get_server_api(args.token, args.site) |
280
|
|
|
aserver_api.check_server() |
281
|
|
|
|
282
|
|
|
if args.user: |
283
|
|
|
username = args.user |
284
|
|
|
else: |
285
|
|
|
user = aserver_api.user() |
286
|
|
|
username = user['login'] |
287
|
|
|
|
288
|
|
|
uploaded_packages = [] |
289
|
|
|
uploaded_projects = [] |
290
|
|
|
|
291
|
|
|
# Flatten file list because of 'windows_glob' function |
292
|
|
|
files = [f for fglob in args.files for f in fglob] |
293
|
|
|
|
294
|
|
|
if args.all: |
295
|
|
|
files += get_convert_files(files) |
296
|
|
|
|
297
|
|
|
for filename in files: |
298
|
|
|
if not exists(filename): |
299
|
|
|
message = 'file %s does not exist' % (filename) |
300
|
|
|
logger.error(message) |
301
|
|
|
raise errors.BinstarError(message) |
302
|
|
|
|
303
|
|
|
package_type = determine_package_type(filename, args) |
304
|
|
|
|
305
|
|
|
if package_type == 'project': |
306
|
|
|
uploaded_projects.append(upload_project(filename, args, username)) |
307
|
|
|
else: |
308
|
|
|
if package_type == 'ipynb' and not args.mode == 'force': |
309
|
|
|
try: |
310
|
|
|
nbformat.read(open(filename), nbformat.NO_CONVERT) |
311
|
|
|
except Exception as error: |
312
|
|
|
logger.error("Invalid notebook file '%s': %s", filename, error) |
313
|
|
|
logger.info("Use --force to upload the file anyways") |
314
|
|
|
continue |
315
|
|
|
|
316
|
|
|
package_info = upload_package( |
317
|
|
|
filename, |
318
|
|
|
package_type=package_type, |
319
|
|
|
aserver_api=aserver_api, |
320
|
|
|
username=username, |
321
|
|
|
args=args) |
322
|
|
|
if package_info: |
323
|
|
|
uploaded_packages.append(package_info) |
324
|
|
|
|
325
|
|
|
for package, upload_info in uploaded_packages: |
326
|
|
|
package_url = upload_info.get('url', 'https://anaconda.org/%s/%s' % (username, package)) |
327
|
|
|
logger.info("{} located at:\n{}\n".format(verbose_package_type(package_type), package_url)) |
328
|
|
|
|
329
|
|
|
for project_name, url in uploaded_projects: |
330
|
|
|
logger.info("Project {} uploaded to {}.\n".format(project_name, url)) |
331
|
|
|
|
332
|
|
|
|
333
|
|
|
def windows_glob(item): |
334
|
|
|
if os.name == 'nt' and '*' in item: |
335
|
|
|
return glob(item) |
336
|
|
|
else: |
337
|
|
|
return [item] |
338
|
|
|
|
339
|
|
|
|
340
|
|
|
def add_parser(subparsers): |
341
|
|
|
description = 'Upload packages to Anaconda Cloud' |
342
|
|
|
parser = subparsers.add_parser('upload', |
343
|
|
|
formatter_class=argparse.RawDescriptionHelpFormatter, |
344
|
|
|
help=description, description=description, |
345
|
|
|
epilog=__doc__) |
346
|
|
|
|
347
|
|
|
parser.add_argument('files', nargs='+', help='Distributions to upload', default=[], type=windows_glob) |
348
|
|
|
|
349
|
|
|
label_help = ( |
350
|
|
|
'{deprecation}Add this file to a specific {label}. ' |
351
|
|
|
'Warning: if the file {label}s do not include "main", ' |
352
|
|
|
'the file will not show up in your user {label}') |
353
|
|
|
|
354
|
|
|
parser.add_argument('-c', '--channel', action='append', default=[], dest='labels', |
355
|
|
|
help=label_help.format(deprecation='[DEPRECATED]\n', label='channel'), |
356
|
|
|
metavar='CHANNELS') |
357
|
|
|
parser.add_argument('-l', '--label', action='append', dest='labels', |
358
|
|
|
help=label_help.format(deprecation='', label='label')) |
359
|
|
|
parser.add_argument('--no-progress', help="Don't show upload progress", action='store_true') |
360
|
|
|
parser.add_argument('-u', '--user', help='User account or Organization, defaults to the current user') |
361
|
|
|
parser.add_argument('--all', help='Use conda convert to generate packages for all platforms and upload them', |
362
|
|
|
action='store_true') |
363
|
|
|
|
364
|
|
|
mgroup = parser.add_argument_group('metadata options') |
365
|
|
|
mgroup.add_argument('-p', '--package', help='Defaults to the package name in the uploaded file') |
366
|
|
|
mgroup.add_argument('-v', '--version', help='Defaults to the package version in the uploaded file') |
367
|
|
|
mgroup.add_argument('-s', '--summary', help='Set the summary of the package') |
368
|
|
|
mgroup.add_argument('-t', '--package-type', help='Set the package type, defaults to autodetect') |
369
|
|
|
mgroup.add_argument('-d', '--description', help='description of the file(s)') |
370
|
|
|
mgroup.add_argument('--thumbnail', help='Notebook\'s thumbnail image') |
371
|
|
|
mgroup.add_argument('--private', help="Create the package with private access", action='store_true') |
372
|
|
|
|
373
|
|
|
register_group = parser.add_mutually_exclusive_group() |
374
|
|
|
register_group.add_argument("--no-register", dest="auto_register", action="store_false", |
375
|
|
|
help='Don\'t create a new package namespace if it does not exist') |
376
|
|
|
register_group.add_argument("--register", dest="auto_register", action="store_true", |
377
|
|
|
help='Create a new package namespace if it does not exist') |
378
|
|
|
parser.set_defaults(auto_register=bool(get_config().get('auto_register', True))) |
379
|
|
|
parser.add_argument('--build-id', help='Anaconda Cloud Build ID (internal only)') |
380
|
|
|
|
381
|
|
|
group = parser.add_mutually_exclusive_group() |
382
|
|
|
group.add_argument('-i', '--interactive', action='store_const', help='Run an interactive prompt if any packages are missing', |
383
|
|
|
dest='mode', const='interactive') |
384
|
|
|
group.add_argument('-f', '--fail', help='Fail if a package or release does not exist (default)', |
385
|
|
|
action='store_const', dest='mode', const='fail') |
386
|
|
|
group.add_argument('--force', help='Force a package upload regardless of errors', |
387
|
|
|
action='store_const', dest='mode', const='force') |
388
|
|
|
|
389
|
|
|
parser.set_defaults(main=main) |
390
|
|
|
|