b2blaze.models.file_list.B2FileList.upload_large_file() - Code Metrics - Inspection of "Add Multithreaded Large File Upload" - sibblegp/b2blaze - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#14)

unknown

created 2018-07-20 21:43 UTC

B2FileList.upload_large_file() B

↳ Parent: b2blaze.models.file_list

Complexity

Conditions

Size

Total Lines	60
Code Lines	44

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	1
CRAP Score	52.0259

Importance

Changes

Metric	Value
cc	7
eloc	44
nop	6
dl	0
loc	60
ccs	1
cts	36
cp	0.0278
crap	52.0259
rs	7.424
c	0
b	0
f	0

How to fix Long Method

"""
Copyright George Sibble 2018
"""

from ..b2_exceptions import B2InvalidBucketName, B2InvalidBucketConfiguration, B2BucketCreationError

from .b2_file import B2File
from ..utilities import b2_url_encode, read_in_chunks, decode_error
from ..b2_exceptions import B2RequestError, B2FileNotFound
from multiprocessing.dummy import Pool as ThreadPool


class B2FileList(object):

    """

    """
    def __init__(self, connector, bucket):
        """

        :param connector:
        :param bucket:
        """
        self.connector = connector
        self.bucket = bucket
        self._files_by_name = {}
        self._files_by_id = {}

    def all(self):
        """

        :return:
        """
        return self._update_files_list(retrieve=True)

    def _update_files_list(self, retrieve=False):
        """

        :param retrieve:
        :return:
        """
        path = '/b2_list_file_names'
        files = []
        new_files_to_retrieve = True
        params = {
            'bucketId': self.bucket.bucket_id,
            'maxFileCount': 10000
        }
        while new_files_to_retrieve:
            response = self.connector.make_request(path=path, method='post', params=params)
            if response.status_code == 200:
                files_json = response.json()
                self._files_by_name = {}
                self._files_by_id = {}
                for file_json in files_json['files']:
                    new_file = B2File(connector=self.connector, parent_list=self, **file_json)
                    files.append(new_file)
                    self._files_by_name[file_json['fileName']] = new_file
                    self._files_by_id[file_json['fileId']] = new_file
                if files_json['nextFileName'] is None:
                    new_files_to_retrieve = False
                else:
                    params['startFileName'] = files_json['nextFileName']
            else:
                raise B2RequestError(decode_error(response))
        if retrieve:
            return files

    def get(self, file_name=None, file_id=None):
        """

        :param file_name:
        :param file_id:
        :return:
        """
        if file_name is not None:
            path = '/b2_list_file_names'
            params = {
                'prefix': b2_url_encode(file_name),
                'bucketId': self.bucket.bucket_id
            }

            response = self.connector.make_request(path, method='post', params=params)
            if response.status_code == 200:
                file_json = response.json()
                if len(file_json['files']) > 0:

                    return B2File(connector=self.connector, parent_list=self, **file_json['files'][0])

                else:
                    raise B2FileNotFound('fileName - ' + file_name)
            else:
                raise B2RequestError(decode_error(response))
        elif file_id is not None:
            path = '/b2_get_file_info'
            params = {
                'fileId': file_id
            }
            response = self.connector.make_request(path, method='post', params=params)
            if response.status_code == 200:
                file_json = response.json()
                return B2File(connector=self.connector, parent_list=self, **file_json)
            else:
                raise B2RequestError(decode_error(response))
        else:
            raise ValueError('file_name or file_id must be passed')

        # self._update_files_list()
        # if file_name is not None:
        #     return self._files_by_name.get(file_name, None)
        # else:
        #     return self._files_by_id.get(file_id, None)
        # pass

    def upload(self, contents, file_name, mime_content_type=None):

        """

        :param contents:
        :param file_name:
        :param mime_content_type:
        :return:
        """
        if file_name[0] == '/':
            file_name = file_name[1:]
        get_upload_url_path = '/b2_get_upload_url'
        params = {
            'bucketId': self.bucket.bucket_id
        }
        upload_url_response = self.connector.make_request(path=get_upload_url_path, method='post', params=params)

        if upload_url_response.status_code == 200:
            upload_url = upload_url_response.json().get('uploadUrl', None)
            auth_token = upload_url_response.json().get('authorizationToken', None)
            upload_response = self.connector.upload_file(file_contents=contents, file_name=file_name,

                                                         upload_url=upload_url, auth_token=auth_token)

            if upload_response.status_code == 200:
                new_file = B2File(connector=self.connector, parent_list=self, **upload_response.json())

                return new_file
            else:
                raise B2RequestError(decode_error(upload_response))
        else:
            raise B2RequestError(decode_error(upload_url_response))

    def upload_large_file(self, contents, file_name, part_size=None, num_threads=4, mime_content_type=None):

        """

        :param contents:
        :param file_name:
        :param part_size:
        :param num_threads:
        :param mime_content_type:
        :return:
        """
        if file_name[0] == '/':
            file_name = file_name[1:]
        if part_size == None:

            part_size = self.connector.recommended_part_size
        start_large_file_path = '/b2_start_large_file'
        params = {
            'bucketId': self.bucket.bucket_id,
            'fileName': b2_url_encode(file_name),
            'contentType': mime_content_type or 'b2/x-auto'
        }
        large_file_response = self.connector.make_request(path=start_large_file_path, method='post', params=params)

        if large_file_response.status_code == 200:
            file_id = large_file_response.json().get('fileId', None)
            get_upload_part_url_path = '/b2_get_upload_part_url'
            params = {
                'fileId': file_id
            }
            pool = ThreadPool(num_threads)
            def thread_woker(args):
class SomeClass:
    def some_method(self):
        """Do x and return foo."""
                part_number = args[0]
                chunk = args[1]
                upload_part_url_response = self.connector.make_request(path=get_upload_part_url_path, method='post', params=params)

                if upload_part_url_response.status_code == 200:
                    upload_url = upload_part_url_response.json().get('uploadUrl')
                    auth_token = upload_part_url_response.json().get('authorizationToken')
                    upload_part_response = self.connector.upload_part(file_contents=chunk, content_length=part_size,

                                                                      part_number=part_number, upload_url=upload_url,

                                                                      auth_token=auth_token)
                    if upload_part_response[0].status_code == 200:
                        return upload_part_response[1]
                    else:
                        raise B2RequestError(decode_error(upload_part_response[0]))
                else:
                    raise B2RequestError(decode_error(upload_part_url_response))
            sha_list = pool.map(thread_woker, enumerate(read_in_chunks(contents, part_size), 1))
            pool.close()
            pool.join()
            finish_large_file_path = '/b2_finish_large_file'
            params = {
                'fileId': file_id,
                'partSha1Array': sha_list
            }
            finish_large_file_response = self.connector.make_request(path=finish_large_file_path, method='post', params=params)

            if finish_large_file_response.status_code == 200:
                new_file = B2File(connector=self.connector, parent_list=self, **finish_large_file_response.json())

                return new_file
            else:
                raise B2RequestError(decode_error(finish_large_file_response))
        else:
            raise B2RequestError(decode_error(large_file_response))


1		"""
2		Copyright George Sibble 2018
3		"""
4
5	1	from ..b2_exceptions import B2InvalidBucketName, B2InvalidBucketConfiguration, B2BucketCreationError
		0 ignored issues – show Unused Code introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report Unused B2InvalidBucketName imported from b2_exceptions Loading history... Unused Code introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report Unused B2BucketCreationError imported from b2_exceptions Loading history... Unused Code introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report Unused B2InvalidBucketConfiguration imported from b2_exceptions Loading history...
6	1	from .b2_file import B2File
7	1	from ..utilities import b2_url_encode, read_in_chunks, decode_error
8	1	from ..b2_exceptions import B2RequestError, B2FileNotFound
9	1	from multiprocessing.dummy import Pool as ThreadPool
		0 ignored issues – show introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report standard import "from multiprocessing.dummy import Pool as ThreadPool" should be placed before "from ..b2_exceptions import B2InvalidBucketName, B2InvalidBucketConfiguration, B2BucketCreationError" Loading history...
10
11	1	class B2FileList(object):
		0 ignored issues – show Documentation introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report Empty class docstring Loading history...
12		"""
13
14		"""
15	1	def __init__(self, connector, bucket):
16		"""
17
18		:param connector:
19		:param bucket:
20		"""
21	1	self.connector = connector
22	1	self.bucket = bucket
23	1	self._files_by_name = {}
24	1	self._files_by_id = {}
25
26	1	def all(self):
27		"""
28
29		:return:
30		"""
31	1	return self._update_files_list(retrieve=True)
32
33	1	def _update_files_list(self, retrieve=False):
34		"""
35
36		:param retrieve:
37		:return:
38		"""
39	1	path = '/b2_list_file_names'
40	1	files = []
41	1	new_files_to_retrieve = True
42	1	params = {
43		'bucketId': self.bucket.bucket_id,
44		'maxFileCount': 10000
45		}
46	1	while new_files_to_retrieve:
47	1	response = self.connector.make_request(path=path, method='post', params=params)
48	1	if response.status_code == 200:
49	1	files_json = response.json()
50	1	self._files_by_name = {}
51	1	self._files_by_id = {}
52	1	for file_json in files_json['files']:
53	1	new_file = B2File(connector=self.connector, parent_list=self, **file_json)
54	1	files.append(new_file)
55	1	self._files_by_name[file_json['fileName']] = new_file
56	1	self._files_by_id[file_json['fileId']] = new_file
57	1	if files_json['nextFileName'] is None:
58	1	new_files_to_retrieve = False
59		else:
60		params['startFileName'] = files_json['nextFileName']
61		else:
62		raise B2RequestError(decode_error(response))
63	1	if retrieve:
64	1	return files
65
66	1	def get(self, file_name=None, file_id=None):
67		"""
68
69		:param file_name:
70		:param file_id:
71		:return:
72		"""
73	1	if file_name is not None:
74	1	path = '/b2_list_file_names'
75	1	params = {
76		'prefix': b2_url_encode(file_name),
77		'bucketId': self.bucket.bucket_id
78		}
79
80	1	response = self.connector.make_request(path, method='post', params=params)
81	1	if response.status_code == 200:
82	1	file_json = response.json()
83	1	if len(file_json['files']) > 0:
		0 ignored issues – show Unused Code introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report Do not use `len(SEQUENCE)` to determine if a sequence is empty Loading history...
84	1	return B2File(connector=self.connector, parent_list=self, **file_json['files'][0])
		0 ignored issues – show Coding Style introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (102/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
85		else:
86	1	raise B2FileNotFound('fileName - ' + file_name)
87		else:
88		raise B2RequestError(decode_error(response))
89	1	elif file_id is not None:
90	1	path = '/b2_get_file_info'
91	1	params = {
92		'fileId': file_id
93		}
94	1	response = self.connector.make_request(path, method='post', params=params)
95	1	if response.status_code == 200:
96	1	file_json = response.json()
97	1	return B2File(connector=self.connector, parent_list=self, **file_json)
98		else:
99	1	raise B2RequestError(decode_error(response))
100		else:
101		raise ValueError('file_name or file_id must be passed')
102
103		# self._update_files_list()
104		# if file_name is not None:
105		# return self._files_by_name.get(file_name, None)
106		# else:
107		# return self._files_by_id.get(file_id, None)
108		# pass
109
110	1	def upload(self, contents, file_name, mime_content_type=None):
		0 ignored issues – show Unused Code introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report The argument `mime_content_type` seems to be unused. Loading history...
111		"""
112
113		:param contents:
114		:param file_name:
115		:param mime_content_type:
116		:return:
117		"""
118	1	if file_name[0] == '/':
119		file_name = file_name[1:]
120	1	get_upload_url_path = '/b2_get_upload_url'
121	1	params = {
122		'bucketId': self.bucket.bucket_id
123		}
124	1	upload_url_response = self.connector.make_request(path=get_upload_url_path, method='post', params=params)
		0 ignored issues – show Coding Style introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (113/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
125	1	if upload_url_response.status_code == 200:
126	1	upload_url = upload_url_response.json().get('uploadUrl', None)
127	1	auth_token = upload_url_response.json().get('authorizationToken', None)
128	1	upload_response = self.connector.upload_file(file_contents=contents, file_name=file_name,
		0 ignored issues – show Coding Style introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (101/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
129		upload_url=upload_url, auth_token=auth_token)
		0 ignored issues – show Coding Style introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (102/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
130	1	if upload_response.status_code == 200:
131	1	new_file = B2File(connector=self.connector, parent_list=self, **upload_response.json())
		0 ignored issues – show Coding Style introduced 2018-06-27 20:20 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (103/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
132	1	return new_file
133		else:
134		raise B2RequestError(decode_error(upload_response))
135		else:
136		raise B2RequestError(decode_error(upload_url_response))
137
138	1	def upload_large_file(self, contents, file_name, part_size=None, num_threads=4, mime_content_type=None):
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (108/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history... best-practice introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report Too many arguments (6/5) Loading history... Comprehensibility introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This function exceeds the maximum number of variables (17/15). Loading history...
139		"""
140
141		:param contents:
142		:param file_name:
143		:param part_size:
144		:param num_threads:
145		:param mime_content_type:
146		:return:
147		"""
148		if file_name[0] == '/':
149		file_name = file_name[1:]
150		if part_size == None:
		0 ignored issues – show introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report Comparison to None should be 'expr is None' Loading history...
151		part_size = self.connector.recommended_part_size
152		start_large_file_path = '/b2_start_large_file'
153		params = {
154		'bucketId': self.bucket.bucket_id,
155		'fileName': b2_url_encode(file_name),
156		'contentType': mime_content_type or 'b2/x-auto'
157		}
158		large_file_response = self.connector.make_request(path=start_large_file_path, method='post', params=params)
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (115/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
159		if large_file_response.status_code == 200:
160		file_id = large_file_response.json().get('fileId', None)
161		get_upload_part_url_path = '/b2_get_upload_part_url'
162		params = {
163		'fileId': file_id
164		}
165		pool = ThreadPool(num_threads)
166		def thread_woker(args):
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This function should have a docstring. The coding style of this project requires that you add a docstring to this code element. Below, you find an example for methods: class SomeClass: def some_method(self): """Do x and return foo.""" If you would like to know more about docstrings, we recommend to read PEP-257: Docstring Conventions. Loading history...
167		part_number = args[0]
168		chunk = args[1]
169		upload_part_url_response = self.connector.make_request(path=get_upload_part_url_path, method='post', params=params)
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (131/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
170		if upload_part_url_response.status_code == 200:
171		upload_url = upload_part_url_response.json().get('uploadUrl')
172		auth_token = upload_part_url_response.json().get('authorizationToken')
173		upload_part_response = self.connector.upload_part(file_contents=chunk, content_length=part_size,
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (116/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
174		part_number=part_number, upload_url=upload_url,
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (117/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
175		auth_token=auth_token)
176		if upload_part_response[0].status_code == 200:
177		return upload_part_response[1]
178		else:
179		raise B2RequestError(decode_error(upload_part_response[0]))
180		else:
181		raise B2RequestError(decode_error(upload_part_url_response))
182		sha_list = pool.map(thread_woker, enumerate(read_in_chunks(contents, part_size), 1))
183		pool.close()
184		pool.join()
185		finish_large_file_path = '/b2_finish_large_file'
186		params = {
187		'fileId': file_id,
188		'partSha1Array': sha_list
189		}
190		finish_large_file_response = self.connector.make_request(path=finish_large_file_path, method='post', params=params)
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (127/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
191		if finish_large_file_response.status_code == 200:
192		new_file = B2File(connector=self.connector, parent_list=self, **finish_large_file_response.json())
		0 ignored issues – show Coding Style introduced 2018-07-20 21:15 UTC by Report Bug Copy Issue Report This line is too long as per the coding-style (114/100). This check looks for lines that are too long. You can specify the maximum line length. Loading history...
193		return new_file
194		else:
195		raise B2RequestError(decode_error(finish_large_file_response))
196		else:
197		raise B2RequestError(decode_error(large_file_response))
198

sibblegp / b2blaze

GitHub Access Token became invalid

Pull Request — master (#14)

B2FileList.upload_large_file() B

Complexity

Size

Duplication

Code Coverage

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like