Completed
Pull Request — develop (#453)
by
unknown
33s
created

upload_package()   D

Complexity

Conditions 9

Size

Total Lines 60

Duplication

Lines 0
Ratio 0 %

Importance

Changes 4
Bugs 0 Features 0
Metric Value
cc 9
c 4
b 0
f 0
dl 0
loc 60
rs 4.9523

How to fix   Long Method   

Long Method

Small methods make your code easier to understand, in particular if combined with a good name. Besides, if your method is small, finding a good name is usually much easier.

For example, if you find yourself adding comments to a method's body, this is usually a good sign to extract the commented part to a new method, and use the comment as a starting point when coming up with a good name for this new method.

Commonly applied refactorings include:

1
'''
2
3
    anaconda upload CONDA_PACKAGE_1.bz2
4
    anaconda upload notebook.ipynb
5
    anaconda upload environment.yml
6
7
##### See Also
8
9
  * [Uploading a Conda Package](http://docs.anaconda.org/using.html#Uploading)
10
  * [Uploading a PyPI Package](http://docs.anaconda.org/using.html#UploadingPypiPackages)
11
12
'''
13
from __future__ import unicode_literals
14
15
import tempfile
16
import argparse
17
import subprocess
18
from glob import glob
19
import logging
20
import os
21
from os.path import exists
22
import sys
23
from collections import defaultdict
24
25
import nbformat
26
27
from binstar_client import errors
28
from binstar_client.utils import bool_input
29
from binstar_client.utils import get_server_api
30
from binstar_client.utils import get_config
31
from binstar_client.utils import upload_print_callback
32
from binstar_client.utils.projects import upload_project
33
from binstar_client.utils.detect import detect_package_type, get_attrs
34
35
36
# Python 3 Support
37
try:
38
    input = raw_input
39
except NameError:
40
    input = input
41
42
43
logger = logging.getLogger('binstar.upload')
44
45
46
PACKAGE_TYPES = defaultdict(lambda: 'Package', {'env': 'Environment', 'ipynb': 'Notebook'})
47
48
49
def verbose_package_type(pkg_type, lowercase=True):
50
    verbose_type = PACKAGE_TYPES[pkg_type]
51
    if lowercase:
52
        verbose_type = verbose_type.lower()
53
    return verbose_type
54
55
56
def create_release(aserver_api, username, package_name, version, release_attrs, announce=None):
57
    aserver_api.add_release(username, package_name, version, [], announce, release_attrs)
58
59
60
def create_release_interactive(aserver_api, username, package_name, version, release_attrs):
61
    logger.info('\nThe release %s/%s/%s does not exist' % (username, package_name, version))
62
    if not bool_input('Would you like to create it now?'):
63
        logger.info('good-bye')
64
        raise SystemExit(-1)
65
66
    description = input('Enter a short description of the release:\n')
67
    logger.info("\nAnnouncements are emailed to your package followers.")
68
    make_announcement = bool_input('Would you like to make an announcement to the package followers?', False)
69
    if make_announcement:
70
        announce = input('Markdown Announcement:\n')
71
    else:
72
        announce = ''
73
74
    aserver_api.add_release(username, package_name, version, [], announce, release_attrs)
75
76
77
def determine_package_type(filename, args):
78
    """
79
    return the file type from the inspected package or from the
80
    -t/--package-type argument
81
    """
82
    if args.package_type:
83
        package_type = args.package_type
84
    else:
85
        logger.info('detecting file type ...')
86
        sys.stdout.flush()
87
        package_type = detect_package_type(filename)
88
        if package_type is None:
89
            message = 'Could not detect package type of file %r please specify package type with option --package-type' % filename
90
            logger.error(message)
91
            raise errors.BinstarError(message)
92
        logger.info(package_type)
93
94
    return package_type
95
96
97
def get_package_name(args, package_attrs, filename, package_type):
98
    if args.package:
99
        if 'name' in package_attrs and package_attrs['name'].lower() != args.package.lower():
100
            msg = 'Package name on the command line " {}" does not match the package name in the file "{}"'.format(
101
                args.package.lower(), package_attrs['name'].lower()
102
            )
103
            logger.error(msg)
104
            raise errors.BinstarError(msg)
105
        package_name = args.package
106
    else:
107
        if 'name' not in package_attrs:
108
            message = "Could not detect package name for package type %s, please use the --package option" % (package_type,)
109
            logger.error(message)
110
            raise errors.BinstarError(message)
111
        package_name = package_attrs['name']
112
113
    return package_name
114
115
116
def get_version(args, release_attrs, package_type):
117
    if args.version:
118
        version = args.version
119
    else:
120
        if 'version' not in release_attrs:
121
            message = "Could not detect package version for package type %s, please use the --version option" % (package_type,)
122
            logger.error(message)
123
            raise errors.BinstarError(message)
124
        version = release_attrs['version']
125
    return version
126
127
128
def add_package(aserver_api, args, username, package_name, package_attrs, package_type):
129
    try:
130
        return aserver_api.package(username, package_name)
131
    except errors.NotFound:
132
        if not args.auto_register:
133
            message = (
134
                'Anaconda Cloud package %s/%s does not exist. '
135
                'Please run "anaconda package --create" to create this package namespace in the cloud.' %
136
                (username, package_name)
137
            )
138
            logger.error(message)
139
            raise errors.UserError(message)
140
        else:
141
142
            if args.summary:
143
                summary = args.summary
144
            else:
145
                if 'summary' not in package_attrs:
146
                    message = "Could not detect package summary for package type %s, please use the --summary option" % (package_type,)
147
                    logger.error(message)
148
                    raise errors.BinstarError(message)
149
                summary = package_attrs['summary']
150
151
            public = not args.private
152
153
            return aserver_api.add_package(
154
                username,
155
                package_name,
156
                summary,
157
                package_attrs.get('license'),
158
                public=public,
159
                attrs=package_attrs,
160
                license_url=package_attrs.get('license_url'),
161
                license_family=package_attrs.get('license_family'),
162
                package_type=package_type,
163
            )
164
165
166
def add_release(aserver_api, args, username, package_name, version, release_attrs):
167
    try:
168
        # Check if the release already exists
169
        aserver_api.release(username, package_name, version)
170
    except errors.NotFound:
171
        if args.mode == 'interactive':
172
            create_release_interactive(aserver_api, username, package_name, version, release_attrs)
173
        else:
174
            create_release(aserver_api, username, package_name, version, release_attrs)
175
176
177
def remove_existing_file(aserver_api, args, username, package_name, version, file_attrs):
178
    try:
179
        aserver_api.distribution(username, package_name, version, file_attrs['basename'])
180
    except errors.NotFound:
181
        return False
182
    else:
183
        if args.mode == 'force':
184
            logger.warning('Distribution %s already exists ... removing' % (file_attrs['basename'],))
185
            aserver_api.remove_dist(username, package_name, version, file_attrs['basename'])
186
        if args.mode == 'interactive':
187
            if bool_input('Distribution %s already exists. Would you like to replace it?' % (file_attrs['basename'],)):
188
                aserver_api.remove_dist(username, package_name, version, file_attrs['basename'])
189
            else:
190
                logger.info('Not replacing distribution %s' % (file_attrs['basename'],))
191
                return True
192
193
194
def upload_package(filename, package_type, aserver_api, username, args):
195
    logger.info('extracting {} attributes for upload ...'.format(verbose_package_type(package_type)))
196
    sys.stdout.flush()
197
    try:
198
        package_attrs, release_attrs, file_attrs = get_attrs(package_type, filename, parser_args=args)
199
    except Exception:
200
        message = 'Trouble reading metadata from {}. Is this a valid {} package?'.format(
201
            filename, verbose_package_type(package_type)
202
        )
203
        logger.error(message)
204
        if args.show_traceback:
205
            raise
206
        raise errors.BinstarError(message)
207
208
    if args.build_id:
209
        file_attrs['attrs']['binstar_build'] = args.build_id
210
211
    logger.info('done')
212
213
    package_name = get_package_name(args, package_attrs, filename, package_type)
214
    version = get_version(args, release_attrs, package_type)
215
216
    package = add_package(aserver_api, args, username, package_name, package_attrs, package_type)
217
    if package_type not in package.get('package_types', []):
218
        package_types = package.get('package_types')
219
        message = 'You already have a {} named \'{}\'. Use a different name for this {}.'.format(
220
            verbose_package_type(package_types[0] if package_types else ''),
221
            package_name,
222
            verbose_package_type(package_type),
223
        )
224
        logger.error(message)
225
        raise errors.BinstarError(message)
226
    add_release(aserver_api, args, username, package_name, version, release_attrs)
227
    binstar_package_type = file_attrs.pop('binstar_package_type', package_type)
228
229
    with open(filename, 'rb') as fd:
230
        logger.info('\nUploading file %s/%s/%s/%s ... ' % (username, package_name, version, file_attrs['basename']))
231
        sys.stdout.flush()
232
233
        if remove_existing_file(aserver_api, args, username, package_name, version, file_attrs):
234
            return None
235
        try:
236
            upload_info = aserver_api.upload(username,
237
                                             package_name,
238
                                             version,
239
                                             file_attrs['basename'],
240
                                             fd, binstar_package_type,
241
                                             args.description,
242
                                             dependencies=file_attrs.get('dependencies'),
243
                                             attrs=file_attrs['attrs'],
244
                                             channels=args.labels,
245
                                             callback=upload_print_callback(args))
246
        except errors.Conflict:
247
            full_name = '%s/%s/%s/%s' % (username, package_name, version, file_attrs['basename'])
248
            logger.info('Distribution already exists. Please use the '
249
                     '-i/--interactive or --force options or `anaconda remove %s`' % full_name)
250
            raise
251
252
        logger.info("\n\nUpload(s) Complete\n")
253
        return [package_name, upload_info]
254
255
256
def get_convert_files(files):
257
    tmpdir = tempfile.mkdtemp()
258
259
    for filepath in files:
260
        logger.info('Running conda convert on %s', filepath)
261
        process = subprocess.Popen(
262
            ['conda-convert', '-p', 'all', filepath, '-o', tmpdir],
263
            stdout=subprocess.PIPE, stderr=subprocess.PIPE
264
        )
265
        stdout, stderr = process.communicate()
266
267
        if stderr:
268
            logger.warning('Couldn\'t generate platform packages for %s: %s', filepath, stderr)
269
270
    result = []
271
    for path, dirs, files in os.walk(tmpdir):
272
        for filename in files:
273
            result.append(os.path.join(path, filename))
274
275
    return result
276
277
278
def main(args):
279
    aserver_api = get_server_api(args.token, args.site)
280
    aserver_api.check_server()
281
282
    if args.user:
283
        username = args.user
284
    else:
285
        user = aserver_api.user()
286
        username = user['login']
287
288
    uploaded_packages = []
289
    uploaded_projects = []
290
291
    # Flatten file list because of 'windows_glob' function
292
    files = [f for fglob in args.files for f in fglob]
293
294
    if args.all:
295
        files += get_convert_files(files)
296
297
    for filename in files:
298
        if not exists(filename):
299
            message = 'file %s does not exist' % (filename)
300
            logger.error(message)
301
            raise errors.BinstarError(message)
302
303
        package_type = determine_package_type(filename, args)
304
305
        if package_type == 'project':
306
            uploaded_projects.append(upload_project(filename, args, username))
307
        else:
308
            if package_type == 'ipynb' and not args.mode == 'force':
309
                try:
310
                    nbformat.read(open(filename), nbformat.NO_CONVERT)
311
                except Exception as error:
312
                    logger.error("Invalid notebook file '%s': %s", filename, error)
313
                    logger.info("Use --force to upload the file anyways")
314
                    continue
315
316
            package_info = upload_package(
317
                filename,
318
                package_type=package_type,
319
                aserver_api=aserver_api,
320
                username=username,
321
                args=args)
322
            if package_info:
323
                uploaded_packages.append(package_info)
324
325
    for package, upload_info in uploaded_packages:
326
        package_url = upload_info.get('url', 'https://anaconda.org/%s/%s' % (username, package))
327
        logger.info("{} located at:\n{}\n".format(verbose_package_type(package_type), package_url))
328
329
    for project_name, url in uploaded_projects:
330
        logger.info("Project {} uploaded to {}.\n".format(project_name, url))
331
332
333
def windows_glob(item):
334
    if os.name == 'nt' and '*' in item:
335
        return glob(item)
336
    else:
337
        return [item]
338
339
340
def add_parser(subparsers):
341
    description = 'Upload packages to Anaconda Cloud'
342
    parser = subparsers.add_parser('upload',
343
                                   formatter_class=argparse.RawDescriptionHelpFormatter,
344
                                   help=description, description=description,
345
                                   epilog=__doc__)
346
347
    parser.add_argument('files', nargs='+', help='Distributions to upload', default=[], type=windows_glob)
348
349
    label_help = (
350
        '{deprecation}Add this file to a specific {label}. '
351
        'Warning: if the file {label}s do not include "main", '
352
        'the file will not show up in your user {label}')
353
354
    parser.add_argument('-c', '--channel', action='append', default=[], dest='labels',
355
                        help=label_help.format(deprecation='[DEPRECATED]\n', label='channel'),
356
                        metavar='CHANNELS')
357
    parser.add_argument('-l', '--label', action='append', dest='labels',
358
                        help=label_help.format(deprecation='', label='label'))
359
    parser.add_argument('--no-progress', help="Don't show upload progress", action='store_true')
360
    parser.add_argument('-u', '--user', help='User account or Organization, defaults to the current user')
361
    parser.add_argument('--all', help='Use conda convert to generate packages for all platforms and upload them',
362
                        action='store_true')
363
364
    mgroup = parser.add_argument_group('metadata options')
365
    mgroup.add_argument('-p', '--package', help='Defaults to the package name in the uploaded file')
366
    mgroup.add_argument('-v', '--version', help='Defaults to the package version in the uploaded file')
367
    mgroup.add_argument('-s', '--summary', help='Set the summary of the package')
368
    mgroup.add_argument('-t', '--package-type', help='Set the package type, defaults to autodetect')
369
    mgroup.add_argument('-d', '--description', help='description of the file(s)')
370
    mgroup.add_argument('--thumbnail', help='Notebook\'s thumbnail image')
371
    mgroup.add_argument('--private', help="Create the package with private access", action='store_true')
372
373
    register_group = parser.add_mutually_exclusive_group()
374
    register_group.add_argument("--no-register", dest="auto_register", action="store_false",
375
                        help='Don\'t create a new package namespace if it does not exist')
376
    register_group.add_argument("--register", dest="auto_register", action="store_true",
377
                        help='Create a new package namespace if it does not exist')
378
    parser.set_defaults(auto_register=bool(get_config().get('auto_register', True)))
379
    parser.add_argument('--build-id', help='Anaconda Cloud Build ID (internal only)')
380
381
    group = parser.add_mutually_exclusive_group()
382
    group.add_argument('-i', '--interactive', action='store_const', help='Run an interactive prompt if any packages are missing',
383
                        dest='mode', const='interactive')
384
    group.add_argument('-f', '--fail', help='Fail if a package or release does not exist (default)',
385
                                        action='store_const', dest='mode', const='fail')
386
    group.add_argument('--force', help='Force a package upload regardless of errors',
387
                                        action='store_const', dest='mode', const='force')
388
389
    parser.set_defaults(main=main)
390