|
1
|
|
|
''' |
|
2
|
|
|
|
|
3
|
|
|
anaconda upload CONDA_PACKAGE_1.bz2 |
|
4
|
|
|
anaconda upload notebook.ipynb |
|
5
|
|
|
anaconda upload environment.yml |
|
6
|
|
|
|
|
7
|
|
|
##### See Also |
|
8
|
|
|
|
|
9
|
|
|
* [Uploading a Conda Package](http://docs.anaconda.org/using.html#Uploading) |
|
10
|
|
|
* [Uploading a PyPI Package](http://docs.anaconda.org/using.html#UploadingPypiPackages) |
|
11
|
|
|
|
|
12
|
|
|
''' |
|
13
|
|
|
from __future__ import unicode_literals |
|
14
|
|
|
|
|
15
|
|
|
import tempfile |
|
16
|
|
|
import argparse |
|
17
|
|
|
import subprocess |
|
18
|
|
|
from glob import glob |
|
19
|
|
|
import logging |
|
20
|
|
|
import os |
|
21
|
|
|
from os.path import exists |
|
22
|
|
|
import sys |
|
23
|
|
|
from collections import defaultdict |
|
24
|
|
|
|
|
25
|
|
|
import nbformat |
|
26
|
|
|
|
|
27
|
|
|
from binstar_client import errors |
|
28
|
|
|
from binstar_client.utils import bool_input |
|
29
|
|
|
from binstar_client.utils import get_server_api |
|
30
|
|
|
from binstar_client.utils import get_config |
|
31
|
|
|
from binstar_client.utils import upload_print_callback |
|
32
|
|
|
from binstar_client.utils.projects import upload_project |
|
33
|
|
|
from binstar_client.utils.detect import detect_package_type, get_attrs |
|
34
|
|
|
|
|
35
|
|
|
|
|
36
|
|
|
# Python 3 Support |
|
37
|
|
|
try: |
|
38
|
|
|
input = raw_input |
|
39
|
|
|
except NameError: |
|
40
|
|
|
input = input |
|
41
|
|
|
|
|
42
|
|
|
|
|
43
|
|
|
logger = logging.getLogger('binstar.upload') |
|
44
|
|
|
|
|
45
|
|
|
|
|
46
|
|
|
PACKAGE_TYPES = defaultdict(lambda: 'Package', {'env': 'Environment', 'ipynb': 'Notebook'}) |
|
47
|
|
|
|
|
48
|
|
|
|
|
49
|
|
|
def verbose_package_type(pkg_type, lowercase=True): |
|
50
|
|
|
verbose_type = PACKAGE_TYPES[pkg_type] |
|
51
|
|
|
if lowercase: |
|
52
|
|
|
verbose_type = verbose_type.lower() |
|
53
|
|
|
return verbose_type |
|
54
|
|
|
|
|
55
|
|
|
|
|
56
|
|
|
def create_release(aserver_api, username, package_name, version, release_attrs, announce=None): |
|
57
|
|
|
aserver_api.add_release(username, package_name, version, [], announce, release_attrs) |
|
58
|
|
|
|
|
59
|
|
|
|
|
60
|
|
|
def create_release_interactive(aserver_api, username, package_name, version, release_attrs): |
|
61
|
|
|
logger.info('\nThe release %s/%s/%s does not exist' % (username, package_name, version)) |
|
62
|
|
|
if not bool_input('Would you like to create it now?'): |
|
63
|
|
|
logger.info('good-bye') |
|
64
|
|
|
raise SystemExit(-1) |
|
65
|
|
|
|
|
66
|
|
|
description = input('Enter a short description of the release:\n') |
|
67
|
|
|
logger.info("\nAnnouncements are emailed to your package followers.") |
|
68
|
|
|
make_announcement = bool_input('Would you like to make an announcement to the package followers?', False) |
|
69
|
|
|
if make_announcement: |
|
70
|
|
|
announce = input('Markdown Announcement:\n') |
|
71
|
|
|
else: |
|
72
|
|
|
announce = '' |
|
73
|
|
|
|
|
74
|
|
|
aserver_api.add_release(username, package_name, version, [], announce, release_attrs) |
|
75
|
|
|
|
|
76
|
|
|
|
|
77
|
|
|
def determine_package_type(filename, args): |
|
78
|
|
|
""" |
|
79
|
|
|
return the file type from the inspected package or from the |
|
80
|
|
|
-t/--package-type argument |
|
81
|
|
|
""" |
|
82
|
|
|
if args.package_type: |
|
83
|
|
|
package_type = args.package_type |
|
84
|
|
|
else: |
|
85
|
|
|
logger.info('detecting file type ...') |
|
86
|
|
|
sys.stdout.flush() |
|
87
|
|
|
package_type = detect_package_type(filename) |
|
88
|
|
|
if package_type is None: |
|
89
|
|
|
message = 'Could not detect package type of file %r please specify package type with option --package-type' % filename |
|
90
|
|
|
logger.error(message) |
|
91
|
|
|
raise errors.BinstarError(message) |
|
92
|
|
|
logger.info(package_type) |
|
93
|
|
|
|
|
94
|
|
|
return package_type |
|
95
|
|
|
|
|
96
|
|
|
|
|
97
|
|
|
def get_package_name(args, package_attrs, filename, package_type): |
|
98
|
|
|
if args.package: |
|
99
|
|
|
if 'name' in package_attrs and package_attrs['name'].lower() != args.package.lower(): |
|
100
|
|
|
msg = 'Package name on the command line " {}" does not match the package name in the file "{}"'.format( |
|
101
|
|
|
args.package.lower(), package_attrs['name'].lower() |
|
102
|
|
|
) |
|
103
|
|
|
logger.error(msg) |
|
104
|
|
|
raise errors.BinstarError(msg) |
|
105
|
|
|
package_name = args.package |
|
106
|
|
|
else: |
|
107
|
|
|
if 'name' not in package_attrs: |
|
108
|
|
|
message = "Could not detect package name for package type %s, please use the --package option" % (package_type,) |
|
109
|
|
|
logger.error(message) |
|
110
|
|
|
raise errors.BinstarError(message) |
|
111
|
|
|
package_name = package_attrs['name'] |
|
112
|
|
|
|
|
113
|
|
|
return package_name |
|
114
|
|
|
|
|
115
|
|
|
|
|
116
|
|
|
def get_version(args, release_attrs, package_type): |
|
117
|
|
|
if args.version: |
|
118
|
|
|
version = args.version |
|
119
|
|
|
else: |
|
120
|
|
|
if 'version' not in release_attrs: |
|
121
|
|
|
message = "Could not detect package version for package type %s, please use the --version option" % (package_type,) |
|
122
|
|
|
logger.error(message) |
|
123
|
|
|
raise errors.BinstarError(message) |
|
124
|
|
|
version = release_attrs['version'] |
|
125
|
|
|
return version |
|
126
|
|
|
|
|
127
|
|
|
|
|
128
|
|
|
def add_package(aserver_api, args, username, package_name, package_attrs, package_type): |
|
129
|
|
|
try: |
|
130
|
|
|
return aserver_api.package(username, package_name) |
|
131
|
|
|
except errors.NotFound: |
|
132
|
|
|
if not args.auto_register: |
|
133
|
|
|
message = ( |
|
134
|
|
|
'Anaconda Cloud package %s/%s does not exist. ' |
|
135
|
|
|
'Please run "anaconda package --create" to create this package namespace in the cloud.' % |
|
136
|
|
|
(username, package_name) |
|
137
|
|
|
) |
|
138
|
|
|
logger.error(message) |
|
139
|
|
|
raise errors.UserError(message) |
|
140
|
|
|
else: |
|
141
|
|
|
|
|
142
|
|
|
if args.summary: |
|
143
|
|
|
summary = args.summary |
|
144
|
|
|
else: |
|
145
|
|
|
if 'summary' not in package_attrs: |
|
146
|
|
|
message = "Could not detect package summary for package type %s, please use the --summary option" % (package_type,) |
|
147
|
|
|
logger.error(message) |
|
148
|
|
|
raise errors.BinstarError(message) |
|
149
|
|
|
summary = package_attrs['summary'] |
|
150
|
|
|
|
|
151
|
|
|
public = not args.private |
|
152
|
|
|
|
|
153
|
|
|
return aserver_api.add_package( |
|
154
|
|
|
username, |
|
155
|
|
|
package_name, |
|
156
|
|
|
summary, |
|
157
|
|
|
package_attrs.get('license'), |
|
158
|
|
|
public=public, |
|
159
|
|
|
attrs=package_attrs, |
|
160
|
|
|
license_url=package_attrs.get('license_url'), |
|
161
|
|
|
license_family=package_attrs.get('license_family'), |
|
162
|
|
|
package_type=package_type, |
|
163
|
|
|
) |
|
164
|
|
|
|
|
165
|
|
|
|
|
166
|
|
|
def add_release(aserver_api, args, username, package_name, version, release_attrs): |
|
167
|
|
|
try: |
|
168
|
|
|
# Check if the release already exists |
|
169
|
|
|
aserver_api.release(username, package_name, version) |
|
170
|
|
|
except errors.NotFound: |
|
171
|
|
|
if args.mode == 'interactive': |
|
172
|
|
|
create_release_interactive(aserver_api, username, package_name, version, release_attrs) |
|
173
|
|
|
else: |
|
174
|
|
|
create_release(aserver_api, username, package_name, version, release_attrs) |
|
175
|
|
|
|
|
176
|
|
|
|
|
177
|
|
|
def remove_existing_file(aserver_api, args, username, package_name, version, file_attrs): |
|
178
|
|
|
try: |
|
179
|
|
|
aserver_api.distribution(username, package_name, version, file_attrs['basename']) |
|
180
|
|
|
except errors.NotFound: |
|
181
|
|
|
return False |
|
182
|
|
|
else: |
|
183
|
|
|
if args.mode == 'force': |
|
184
|
|
|
logger.warning('Distribution %s already exists ... removing' % (file_attrs['basename'],)) |
|
185
|
|
|
aserver_api.remove_dist(username, package_name, version, file_attrs['basename']) |
|
186
|
|
|
if args.mode == 'interactive': |
|
187
|
|
|
if bool_input('Distribution %s already exists. Would you like to replace it?' % (file_attrs['basename'],)): |
|
188
|
|
|
aserver_api.remove_dist(username, package_name, version, file_attrs['basename']) |
|
189
|
|
|
else: |
|
190
|
|
|
logger.info('Not replacing distribution %s' % (file_attrs['basename'],)) |
|
191
|
|
|
return True |
|
192
|
|
|
|
|
193
|
|
|
|
|
194
|
|
|
def upload_package(filename, package_type, aserver_api, username, args): |
|
195
|
|
|
logger.info('extracting {} attributes for upload ...'.format(verbose_package_type(package_type))) |
|
196
|
|
|
sys.stdout.flush() |
|
197
|
|
|
try: |
|
198
|
|
|
package_attrs, release_attrs, file_attrs = get_attrs(package_type, filename, parser_args=args) |
|
199
|
|
|
except Exception: |
|
200
|
|
|
message = 'Trouble reading metadata from {}. Is this a valid {} package?'.format( |
|
201
|
|
|
filename, verbose_package_type(package_type) |
|
202
|
|
|
) |
|
203
|
|
|
logger.error(message) |
|
204
|
|
|
if args.show_traceback: |
|
205
|
|
|
raise |
|
206
|
|
|
raise errors.BinstarError(message) |
|
207
|
|
|
|
|
208
|
|
|
if args.build_id: |
|
209
|
|
|
file_attrs['attrs']['binstar_build'] = args.build_id |
|
210
|
|
|
|
|
211
|
|
|
logger.info('done') |
|
212
|
|
|
|
|
213
|
|
|
package_name = get_package_name(args, package_attrs, filename, package_type) |
|
214
|
|
|
version = get_version(args, release_attrs, package_type) |
|
215
|
|
|
|
|
216
|
|
|
package = add_package(aserver_api, args, username, package_name, package_attrs, package_type) |
|
217
|
|
|
if package_type not in package.get('package_types', []): |
|
218
|
|
|
package_types = package.get('package_types') |
|
219
|
|
|
message = 'You already have a {} named \'{}\'. Use a different name for this {}.'.format( |
|
220
|
|
|
verbose_package_type(package_types[0] if package_types else ''), |
|
221
|
|
|
package_name, |
|
222
|
|
|
verbose_package_type(package_type), |
|
223
|
|
|
) |
|
224
|
|
|
logger.error(message) |
|
225
|
|
|
raise errors.BinstarError(message) |
|
226
|
|
|
add_release(aserver_api, args, username, package_name, version, release_attrs) |
|
227
|
|
|
binstar_package_type = file_attrs.pop('binstar_package_type', package_type) |
|
228
|
|
|
|
|
229
|
|
|
with open(filename, 'rb') as fd: |
|
230
|
|
|
logger.info('\nUploading file %s/%s/%s/%s ... ' % (username, package_name, version, file_attrs['basename'])) |
|
231
|
|
|
sys.stdout.flush() |
|
232
|
|
|
|
|
233
|
|
|
if remove_existing_file(aserver_api, args, username, package_name, version, file_attrs): |
|
234
|
|
|
return None |
|
235
|
|
|
try: |
|
236
|
|
|
upload_info = aserver_api.upload(username, |
|
237
|
|
|
package_name, |
|
238
|
|
|
version, |
|
239
|
|
|
file_attrs['basename'], |
|
240
|
|
|
fd, binstar_package_type, |
|
241
|
|
|
args.description, |
|
242
|
|
|
dependencies=file_attrs.get('dependencies'), |
|
243
|
|
|
attrs=file_attrs['attrs'], |
|
244
|
|
|
channels=args.labels, |
|
245
|
|
|
callback=upload_print_callback(args)) |
|
246
|
|
|
except errors.Conflict: |
|
247
|
|
|
full_name = '%s/%s/%s/%s' % (username, package_name, version, file_attrs['basename']) |
|
248
|
|
|
logger.info('Distribution already exists. Please use the ' |
|
249
|
|
|
'-i/--interactive or --force options or `anaconda remove %s`' % full_name) |
|
250
|
|
|
raise |
|
251
|
|
|
|
|
252
|
|
|
logger.info("\n\nUpload(s) Complete\n") |
|
253
|
|
|
return [package_name, upload_info] |
|
254
|
|
|
|
|
255
|
|
|
|
|
256
|
|
|
def get_convert_files(files): |
|
257
|
|
|
tmpdir = tempfile.mkdtemp() |
|
258
|
|
|
|
|
259
|
|
|
for filepath in files: |
|
260
|
|
|
logger.info('Running conda convert on %s', filepath) |
|
261
|
|
|
process = subprocess.Popen( |
|
262
|
|
|
['conda-convert', '-p', 'all', filepath, '-o', tmpdir], |
|
263
|
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE |
|
264
|
|
|
) |
|
265
|
|
|
stdout, stderr = process.communicate() |
|
266
|
|
|
|
|
267
|
|
|
if stderr: |
|
268
|
|
|
logger.warning('Couldn\'t generate platform packages for %s: %s', filepath, stderr) |
|
269
|
|
|
|
|
270
|
|
|
result = [] |
|
271
|
|
|
for path, dirs, files in os.walk(tmpdir): |
|
272
|
|
|
for filename in files: |
|
273
|
|
|
result.append(os.path.join(path, filename)) |
|
274
|
|
|
|
|
275
|
|
|
return result |
|
276
|
|
|
|
|
277
|
|
|
|
|
278
|
|
|
def main(args): |
|
279
|
|
|
aserver_api = get_server_api(args.token, args.site) |
|
280
|
|
|
aserver_api.check_server() |
|
281
|
|
|
|
|
282
|
|
|
if args.user: |
|
283
|
|
|
username = args.user |
|
284
|
|
|
else: |
|
285
|
|
|
user = aserver_api.user() |
|
286
|
|
|
username = user['login'] |
|
287
|
|
|
|
|
288
|
|
|
uploaded_packages = [] |
|
289
|
|
|
uploaded_projects = [] |
|
290
|
|
|
|
|
291
|
|
|
# Flatten file list because of 'windows_glob' function |
|
292
|
|
|
files = [f for fglob in args.files for f in fglob] |
|
293
|
|
|
|
|
294
|
|
|
if args.all: |
|
295
|
|
|
files += get_convert_files(files) |
|
296
|
|
|
|
|
297
|
|
|
for filename in files: |
|
298
|
|
|
if not exists(filename): |
|
299
|
|
|
message = 'file %s does not exist' % (filename) |
|
300
|
|
|
logger.error(message) |
|
301
|
|
|
raise errors.BinstarError(message) |
|
302
|
|
|
|
|
303
|
|
|
package_type = determine_package_type(filename, args) |
|
304
|
|
|
|
|
305
|
|
|
if package_type == 'project': |
|
306
|
|
|
uploaded_projects.append(upload_project(filename, args, username)) |
|
307
|
|
|
else: |
|
308
|
|
|
if package_type == 'ipynb' and not args.mode == 'force': |
|
309
|
|
|
try: |
|
310
|
|
|
nbformat.read(open(filename), nbformat.NO_CONVERT) |
|
311
|
|
|
except Exception as error: |
|
312
|
|
|
logger.error("Invalid notebook file '%s': %s", filename, error) |
|
313
|
|
|
logger.info("Use --force to upload the file anyways") |
|
314
|
|
|
continue |
|
315
|
|
|
|
|
316
|
|
|
package_info = upload_package( |
|
317
|
|
|
filename, |
|
318
|
|
|
package_type=package_type, |
|
319
|
|
|
aserver_api=aserver_api, |
|
320
|
|
|
username=username, |
|
321
|
|
|
args=args) |
|
322
|
|
|
if package_info: |
|
323
|
|
|
uploaded_packages.append(package_info) |
|
324
|
|
|
|
|
325
|
|
|
for package, upload_info in uploaded_packages: |
|
326
|
|
|
package_url = upload_info.get('url', 'https://anaconda.org/%s/%s' % (username, package)) |
|
327
|
|
|
logger.info("{} located at:\n{}\n".format(verbose_package_type(package_type), package_url)) |
|
328
|
|
|
|
|
329
|
|
|
for project_name, url in uploaded_projects: |
|
330
|
|
|
logger.info("Project {} uploaded to {}.\n".format(project_name, url)) |
|
331
|
|
|
|
|
332
|
|
|
|
|
333
|
|
|
def windows_glob(item): |
|
334
|
|
|
if os.name == 'nt' and '*' in item: |
|
335
|
|
|
return glob(item) |
|
336
|
|
|
else: |
|
337
|
|
|
return [item] |
|
338
|
|
|
|
|
339
|
|
|
|
|
340
|
|
|
def add_parser(subparsers): |
|
341
|
|
|
description = 'Upload packages to Anaconda Cloud' |
|
342
|
|
|
parser = subparsers.add_parser('upload', |
|
343
|
|
|
formatter_class=argparse.RawDescriptionHelpFormatter, |
|
344
|
|
|
help=description, description=description, |
|
345
|
|
|
epilog=__doc__) |
|
346
|
|
|
|
|
347
|
|
|
parser.add_argument('files', nargs='+', help='Distributions to upload', default=[], type=windows_glob) |
|
348
|
|
|
|
|
349
|
|
|
label_help = ( |
|
350
|
|
|
'{deprecation}Add this file to a specific {label}. ' |
|
351
|
|
|
'Warning: if the file {label}s do not include "main", ' |
|
352
|
|
|
'the file will not show up in your user {label}') |
|
353
|
|
|
|
|
354
|
|
|
parser.add_argument('-c', '--channel', action='append', default=[], dest='labels', |
|
355
|
|
|
help=label_help.format(deprecation='[DEPRECATED]\n', label='channel'), |
|
356
|
|
|
metavar='CHANNELS') |
|
357
|
|
|
parser.add_argument('-l', '--label', action='append', dest='labels', |
|
358
|
|
|
help=label_help.format(deprecation='', label='label')) |
|
359
|
|
|
parser.add_argument('--no-progress', help="Don't show upload progress", action='store_true') |
|
360
|
|
|
parser.add_argument('-u', '--user', help='User account or Organization, defaults to the current user') |
|
361
|
|
|
parser.add_argument('--all', help='Use conda convert to generate packages for all platforms and upload them', |
|
362
|
|
|
action='store_true') |
|
363
|
|
|
|
|
364
|
|
|
mgroup = parser.add_argument_group('metadata options') |
|
365
|
|
|
mgroup.add_argument('-p', '--package', help='Defaults to the package name in the uploaded file') |
|
366
|
|
|
mgroup.add_argument('-v', '--version', help='Defaults to the package version in the uploaded file') |
|
367
|
|
|
mgroup.add_argument('-s', '--summary', help='Set the summary of the package') |
|
368
|
|
|
mgroup.add_argument('-t', '--package-type', help='Set the package type, defaults to autodetect') |
|
369
|
|
|
mgroup.add_argument('-d', '--description', help='description of the file(s)') |
|
370
|
|
|
mgroup.add_argument('--thumbnail', help='Notebook\'s thumbnail image') |
|
371
|
|
|
mgroup.add_argument('--private', help="Create the package with private access", action='store_true') |
|
372
|
|
|
|
|
373
|
|
|
register_group = parser.add_mutually_exclusive_group() |
|
374
|
|
|
register_group.add_argument("--no-register", dest="auto_register", action="store_false", |
|
375
|
|
|
help='Don\'t create a new package namespace if it does not exist') |
|
376
|
|
|
register_group.add_argument("--register", dest="auto_register", action="store_true", |
|
377
|
|
|
help='Create a new package namespace if it does not exist') |
|
378
|
|
|
parser.set_defaults(auto_register=bool(get_config().get('auto_register', True))) |
|
379
|
|
|
parser.add_argument('--build-id', help='Anaconda Cloud Build ID (internal only)') |
|
380
|
|
|
|
|
381
|
|
|
group = parser.add_mutually_exclusive_group() |
|
382
|
|
|
group.add_argument('-i', '--interactive', action='store_const', help='Run an interactive prompt if any packages are missing', |
|
383
|
|
|
dest='mode', const='interactive') |
|
384
|
|
|
group.add_argument('-f', '--fail', help='Fail if a package or release does not exist (default)', |
|
385
|
|
|
action='store_const', dest='mode', const='fail') |
|
386
|
|
|
group.add_argument('--force', help='Force a package upload regardless of errors', |
|
387
|
|
|
action='store_const', dest='mode', const='force') |
|
388
|
|
|
|
|
389
|
|
|
parser.set_defaults(main=main) |
|
390
|
|
|
|