1
|
|
|
from __future__ import print_function, absolute_import, unicode_literals |
2
|
|
|
|
3
|
|
|
import base64 |
4
|
|
|
import json |
5
|
|
|
import logging |
6
|
|
|
import os |
7
|
|
|
import sys |
8
|
|
|
import time |
9
|
|
|
|
10
|
|
|
from hashlib import md5 |
11
|
|
|
|
12
|
|
|
# re-export parse_version |
13
|
|
|
from pkg_resources import parse_version as pv |
14
|
|
|
from .spec import PackageSpec, package_specs, parse_specs |
15
|
|
|
|
16
|
|
|
# Re-export config |
17
|
|
|
from .config import (get_server_api, dirs, load_token, store_token, |
18
|
|
|
remove_token, get_config, set_config, load_config, |
19
|
|
|
get_binstar, |
20
|
|
|
USER_CONFIG, USER_LOGDIR, SITE_CONFIG, DEFAULT_CONFIG) |
21
|
|
|
|
22
|
|
|
from six.moves import input |
23
|
|
|
|
24
|
|
|
|
25
|
|
|
logger = logging.getLogger('binstar') |
26
|
|
|
|
27
|
|
|
|
28
|
|
|
def jencode(*E, **F): |
29
|
|
|
payload = dict(*E, **F) |
30
|
|
|
return json.dumps(payload), {'Content-Type': 'application/json'} |
31
|
|
|
|
32
|
|
|
|
33
|
|
|
def compute_hash(fp, buf_size=8192, size=None, hash_algorithm=md5): |
34
|
|
|
hash_obj = hash_algorithm() |
35
|
|
|
spos = fp.tell() |
36
|
|
|
if size and size < buf_size: |
37
|
|
|
s = fp.read(size) |
38
|
|
|
else: |
39
|
|
|
s = fp.read(buf_size) |
40
|
|
|
while s: |
41
|
|
|
hash_obj.update(s) |
42
|
|
|
if size: |
43
|
|
|
size -= len(s) |
44
|
|
|
if size <= 0: |
45
|
|
|
break |
46
|
|
|
if size and size < buf_size: |
47
|
|
|
s = fp.read(size) |
48
|
|
|
else: |
49
|
|
|
s = fp.read(buf_size) |
50
|
|
|
hex_digest = hash_obj.hexdigest() |
51
|
|
|
|
52
|
|
|
b64encode = getattr(base64, 'encodebytes', base64.encodestring) |
53
|
|
|
base64_digest = b64encode(hash_obj.digest()) |
54
|
|
|
if base64_digest[-1] == '\n': |
55
|
|
|
base64_digest = base64_digest[0:-1] |
56
|
|
|
# data_size based on bytes read. |
57
|
|
|
data_size = fp.tell() - spos |
58
|
|
|
fp.seek(spos) |
59
|
|
|
return (hex_digest, base64_digest, data_size) |
60
|
|
|
|
61
|
|
|
|
62
|
|
|
class upload_in_chunks(object): |
63
|
|
|
def __init__(self, fd, chunksize=1 << 13): |
64
|
|
|
self.fd = fd |
65
|
|
|
self.chunksize = chunksize |
66
|
|
|
self.totalsize = os.fstat(fd.fileno()).st_size |
67
|
|
|
self.readsofar = 0 |
68
|
|
|
|
69
|
|
|
def __iter__(self): |
70
|
|
|
sys.stderr.write('Progress:\n') |
71
|
|
|
while True: |
72
|
|
|
data = self.fd.read(self.chunksize) |
73
|
|
|
if not data: |
74
|
|
|
sys.stderr.write("\n") |
75
|
|
|
break |
76
|
|
|
self.readsofar += len(data) |
77
|
|
|
percent = self.readsofar * 1e2 / self.totalsize |
78
|
|
|
sys.stderr.write("\r{percent:3.0f}%".format(percent=percent)) |
79
|
|
|
yield data |
80
|
|
|
|
81
|
|
|
def __len__(self): |
82
|
|
|
return self.totalsize |
83
|
|
|
|
84
|
|
|
|
85
|
|
|
def upload_with_progress(fd): |
86
|
|
|
it = upload_in_chunks(fd) |
87
|
|
|
IterableToFileAdapter(it) |
88
|
|
|
|
89
|
|
|
|
90
|
|
|
class IterableToFileAdapter(object): |
91
|
|
|
def __init__(self, iterable): |
92
|
|
|
self.iterator = iter(iterable) |
93
|
|
|
self.length = len(iterable) |
94
|
|
|
|
95
|
|
|
def read(self, size=-1): # TBD: add buffer for `len(data) > size` case |
96
|
|
|
return next(self.iterator, b'') |
97
|
|
|
|
98
|
|
|
def __len__(self): |
99
|
|
|
return self.length |
100
|
|
|
|
101
|
|
|
|
102
|
|
|
def bool_input(prompt, default=True): |
103
|
|
|
default_str = '[Y|n]' if default else '[y|N]' |
104
|
|
|
while 1: |
105
|
|
|
inpt = input('%s %s: ' % (prompt, default_str)) |
106
|
|
|
if inpt.lower() in ['y', 'yes'] and not default: |
107
|
|
|
return True |
108
|
|
|
elif inpt.lower() in ['', 'n', 'no'] and not default: |
109
|
|
|
return False |
110
|
|
|
elif inpt.lower() in ['', 'y', 'yes']: |
111
|
|
|
return True |
112
|
|
|
elif inpt.lower() in ['n', 'no']: |
113
|
|
|
return False |
114
|
|
|
else: |
115
|
|
|
sys.stderr.write('please enter yes or no\n') |
116
|
|
|
|
117
|
|
|
|
118
|
|
|
WAIT_SECONDS = 15 |
119
|
|
|
|
120
|
|
|
|
121
|
|
|
def upload_print_callback(args): |
122
|
|
|
start_time = time.time() |
123
|
|
|
if args.no_progress or args.log_level > logging.INFO: |
124
|
|
|
|
125
|
|
|
def callback(curr, total): |
126
|
|
|
perc = 100.0 * curr / total if total else 0 |
127
|
|
|
|
128
|
|
|
if (time.time() - callback.last_output) > WAIT_SECONDS: |
129
|
|
|
print('| %.2f%% ' % (perc), end='') |
130
|
|
|
sys.stdout.flush() |
131
|
|
|
callback.last_output = time.time() |
132
|
|
|
|
133
|
|
|
callback.last_output = time.time() |
134
|
|
|
|
135
|
|
|
return callback |
136
|
|
|
|
137
|
|
|
def callback(curr, total): |
138
|
|
|
curr_time = time.time() |
139
|
|
|
time_delta = curr_time - start_time |
140
|
|
|
|
141
|
|
|
remain = total - curr |
142
|
|
|
if curr and remain: |
143
|
|
|
eta = 1.0 * time_delta / curr * remain / 60.0 |
144
|
|
|
else: |
145
|
|
|
eta = 0 |
146
|
|
|
|
147
|
|
|
curr_kb = curr // 1024 |
148
|
|
|
total_kb = total // 1024 |
149
|
|
|
perc = 100.0 * curr / total if total else 0 |
150
|
|
|
|
151
|
|
|
msg = '\r uploaded %(curr_kb)i of %(total_kb)iKb: %(perc).2f%% ETA: %(eta).1f minutes' |
152
|
|
|
sys.stderr.write(msg % locals()) |
153
|
|
|
sys.stderr.flush() |
154
|
|
|
if curr == total: |
155
|
|
|
sys.stderr.write('\n') |
156
|
|
|
|
157
|
|
|
return callback |
158
|
|
|
|