doorstop.core.exporter._tabulate() - Code Metrics - jacebrowning/doorstop - Measure and Improve Code Quality continuously with Scrutinizer

doorstop.core.exporter._tabulate() D
last analyzed 2019-08-12 00:16 UTC

↳ Parent: doorstop.core.exporter

Complexity

Conditions

Size

Total Lines	47
Code Lines	27

Duplication

Lines	0
Ratio	0 %

Code Coverage

Tests	26
CRAP Score	13

Importance

Changes

Metric	Value
eloc	27
dl	0
loc	47
ccs	26
cts	26
cp	1
rs	4.2
c	0
b	0
f	0
cc	13
nop	3
crap	13

How to fix Complexity

# SPDX-License-Identifier: LGPL-3.0-only

"""Functions to export documents and items."""

import csv
import datetime
import os
from collections import defaultdict
from typing import Any, Dict

import openpyxl
import yaml

from doorstop import common, settings
from doorstop.common import DoorstopError
from doorstop.core.types import iter_documents, iter_items

LIST_SEP = '\n'  # string separating list values when joined in a string

XLSX_MAX_WIDTH = 65.0  # maximum width for a column
XLSX_FILTER_PADDING = 3.5  # column padding to account for filter button

log = common.logger(__name__)


def export(obj, path, ext=None, **kwargs):
    """Export an object to a given format.

    The function can be called in two ways:

    1. document or item-like object + output file path
    2. tree-like object + output directory path

    :param obj: (1) Item, list of Items, Document or (2) Tree
    :param path: (1) output file path or (2) output directory path
    :param ext: file extension to override output extension

    :raises: :class:`doorstop.common.DoorstopError` for unknown file formats

    :return: output location if files created, else None

    """
    # Determine the output format
    ext = ext or os.path.splitext(path)[-1] or '.csv'
    check(ext)

    # Export documents
    count = 0
    for obj2, path2 in iter_documents(obj, path, ext):
        count += 1

        # Export content to the specified path
        common.create_dirname(path2)
        log.info("exporting to {}...".format(path2))
        if ext in FORMAT_LINES:
            lines = export_lines(obj2, ext, **kwargs)
            common.write_lines(lines, path2)
        else:
            export_file(obj2, path2, ext, **kwargs)

    # Return the exported path
    if count:
        msg = "exported to {} file{}".format(count, 's' if count > 1 else '')
        log.info(msg)
        return path
    else:
        log.warning("nothing to export")
        return None


def export_lines(obj, ext='.yml', **kwargs):
    """Yield lines for an export in the specified format.

    :param obj: Item, list of Items, or Document to export
    :param ext: file extension to specify the output format

    :raises: :class:`doorstop.common.DoorstopError` for unknown file formats

    :return: lines generator

    """
    gen = check(ext, get_lines_gen=True)
    log.debug("yielding {} as lines of {}...".format(obj, ext))
    yield from gen(obj, **kwargs)


def export_file(obj, path, ext=None, **kwargs):
    """Create a file object for an export in the specified format.

    :param obj: Item, list of Items, or Document to export
    :param path: output file location with desired extension
    :param ext: file extension to override output path's extension

    :raises: :class:`doorstop.common.DoorstopError` for unknown file formats

    :return: path to created file

    """
    ext = ext or os.path.splitext(path)[-1]
    func = check(ext, get_file_func=True)
    log.debug("converting %s to file format %s...", obj, ext)
    try:
        return func(obj, path, **kwargs)
    except IOError:
        msg = "unable to write to: {}".format(path)
        raise common.DoorstopFileError(msg) from None


def _lines_yaml(obj, **_):
    """Yield lines for a YAML export.

    :param obj: Item, list of Items, or Document to export

    :return: iterator of lines of text

    """
    for item in iter_items(obj):

        data = {str(item.uid): item.data}
        text = yaml.dump(data, default_flow_style=False, allow_unicode=True)
        yield text


def _tabulate(obj, sep=LIST_SEP, auto=False):
    """Yield lines of header/data for tabular export.

    :param obj: Item, list of Items, or Document to export
    :param sep: string separating list values when joined in a string
    :param auto: include placeholders for new items on import

    :return: iterator of rows of data

    """
    yield_header = True

    for item in iter_items(obj):

        data = item.data

        # Yield header
        if yield_header:
            header = ['level', 'text', 'ref', 'links']
            for value in sorted(data.keys()):
                if value not in header:
                    header.append(value)
            yield ['uid'] + header
            yield_header = False

        # Yield row
        row = [item.uid]
        for key in header:

            value = data.get(key)
            if key == 'level':
                # some levels are floats for YAML presentation
                value = str(value)
            elif key == 'links':
                # separate identifiers with a delimiter
                value = sep.join(uid.string for uid in item.links)
            elif isinstance(value, str) and key not in ('reviewed',):
                # remove sentence boundaries and line wrapping
                value = item.get(key)
            elif value is None:
                value = ''
            row.append(value)
        yield row

    # Yield placeholders for new items
    if auto:
        for _ in range(settings.PLACEHOLDER_COUNT):
            yield [settings.PLACEHOLDER]


def _file_csv(obj, path, delimiter=',', auto=False):
    """Create a CSV file at the given path.

    :param obj: Item, list of Items, or Document to export
    :param path: location to export CSV file
    :param delimiter: character to delimit fields
    :param auto: include placeholders for new items on import

    :return: path of created file

    """
    with open(path, 'w', newline='', encoding='utf-8') as stream:
        writer = csv.writer(stream, delimiter=delimiter)
        for row in _tabulate(obj, auto=auto):
            writer.writerow(row)
    return path


def _file_tsv(obj, path, auto=False):
    """Create a TSV file at the given path.

    :param obj: Item, list of Items, or Document to export
    :param path: location to export TSV file
    :param auto: include placeholders for new items on import

    :return: path of created file

    """
    return _file_csv(obj, path, delimiter='\t', auto=auto)


def _file_xlsx(obj, path, auto=False):
    """Create an XLSX file at the given path.

    :param obj: Item, list of Items, or Document to export
    :param path: location to export XLSX file
    :param auto: include placeholders for new items on import

    :return: path of created file

    """
    workbook = _get_xlsx(obj, auto)
    workbook.save(path)

    return path


def _get_xlsx(obj, auto):
    """Create an XLSX workbook object.

    :param obj: Item, list of Items, or Document to export
    :param auto: include placeholders for new items on import

    :return: new workbook

    """
    col_widths: Dict[Any, float] = defaultdict(float)
    col = 'A'

    # Create a new workbook
    workbook = openpyxl.Workbook()
    worksheet = workbook.active

    # Populate cells
    for row, data in enumerate(_tabulate(obj, auto=auto), start=1):
        for col_idx, value in enumerate(data, start=1):
            cell = worksheet.cell(column=col_idx, row=row)

            # wrap text in every cell
            alignment = openpyxl.styles.Alignment(
                vertical='top', horizontal='left', wrap_text=True
            )
            cell.alignment = alignment
            # and bold header rows
            if row == 1:
                cell.font = openpyxl.styles.Font(bold=True)

            # convert incompatible Excel types:
            # http://pythonhosted.org/openpyxl/api.html#openpyxl.cell.Cell.value
            if isinstance(value, (int, float, datetime.datetime)):
                cell.value = value
            else:
                cell.value = str(value)

            # track cell width
            col_widths[col_idx] = max(col_widths[col_idx], _width(str(value)))

    # Add filter up to the last column
    col_letter = openpyxl.utils.get_column_letter(len(col_widths))
    worksheet.auto_filter.ref = "A1:%s1" % col_letter

    # Set column width based on column contents
    for col in col_widths:
        if col_widths[col] > XLSX_MAX_WIDTH:
            width = XLSX_MAX_WIDTH
        else:
            width = col_widths[col] + XLSX_FILTER_PADDING
        col_letter = openpyxl.utils.get_column_letter(col)
        worksheet.column_dimensions[col_letter].width = width

    # Freeze top row
    worksheet.freeze_panes = worksheet.cell(row=2, column=1)

    return workbook


def _width(text):
    """Get the maximum length in a multiline string."""
    if text:
        return max(len(line) for line in text.splitlines())
    else:
        return 0


# Mapping from file extension to lines generator
FORMAT_LINES = {'.yml': _lines_yaml}
# Mapping from file extension to file generator
FORMAT_FILE = {'.csv': _file_csv, '.tsv': _file_tsv, '.xlsx': _file_xlsx}
# Union of format dictionaries
FORMAT = dict(list(FORMAT_LINES.items()) + list(FORMAT_FILE.items()))  # type: ignore


def check(ext, get_lines_gen=False, get_file_func=False):
    """Confirm an extension is supported for export.

    :param get_lines_func: return a lines generator if available
    :param get_file_func: return a file creator if available

    :raises: :class:`doorstop.common.DoorstopError` for unknown formats

    :return: function requested if available

    """
    exts = ', '.join(ext for ext in FORMAT)
    lines_exts = ', '.join(ext for ext in FORMAT_LINES)
    file_exts = ', '.join(ext for ext in FORMAT_FILE)
    fmt = "unknown {{}} format: {} (options: {{}})".format(ext or None)

    if get_lines_gen:
        try:
            gen = FORMAT_LINES[ext]
        except KeyError:
            exc = DoorstopError(fmt.format("lines export", lines_exts))
            raise exc from None
        else:
            log.debug("found lines generator for: {}".format(ext))
            return gen

    if get_file_func:
        try:
            func = FORMAT_FILE[ext]
        except KeyError:
            exc = DoorstopError(fmt.format("file export", file_exts))
            raise exc from None
        else:
            log.debug("found file creator for: {}".format(ext))
            return func

    if ext not in FORMAT:
        exc = DoorstopError(fmt.format("export", exts))
        raise exc

    return None


1		# SPDX-License-Identifier: LGPL-3.0-only
2
3	1	"""Functions to export documents and items."""
4	1
5	1	import csv
6	1	import datetime
7		import os
8	1	from collections import defaultdict
9	1	from typing import Any, Dict
10
11	1	import openpyxl
12	1	import yaml
13	1
14	1	from doorstop import common, settings
15		from doorstop.common import DoorstopError
16	1	from doorstop.core.types import iter_documents, iter_items
17
18	1	LIST_SEP = '\n' # string separating list values when joined in a string
19	1
20		XLSX_MAX_WIDTH = 65.0 # maximum width for a column
21	1	XLSX_FILTER_PADDING = 3.5 # column padding to account for filter button
22
23		log = common.logger(__name__)
24	1
25
26		def export(obj, path, ext=None, **kwargs):
27		"""Export an object to a given format.
28
29		The function can be called in two ways:
30
31		1. document or item-like object + output file path
32		2. tree-like object + output directory path
33
34		:param obj: (1) Item, list of Items, Document or (2) Tree
35		:param path: (1) output file path or (2) output directory path
36		:param ext: file extension to override output extension
37
38		:raises: :class:`doorstop.common.DoorstopError` for unknown file formats
39
40		:return: output location if files created, else None
41
42	1	"""
43	1	# Determine the output format
44		ext = ext or os.path.splitext(path)[-1] or '.csv'
45		check(ext)
46	1
47	1	# Export documents
48	1	count = 0
49		for obj2, path2 in iter_documents(obj, path, ext):
50		count += 1
51	1
52	1	# Export content to the specified path
53	1	common.create_dirname(path2)
54	1	log.info("exporting to {}...".format(path2))
55	1	if ext in FORMAT_LINES:
56		lines = export_lines(obj2, ext, **kwargs)
57	1	common.write_lines(lines, path2)
58		else:
59		export_file(obj2, path2, ext, **kwargs)
60	1
61	1	# Return the exported path
62	1	if count:
63	1	msg = "exported to {} file{}".format(count, 's' if count > 1 else '')
64		log.info(msg)
65	1	return path
66	1	else:
67		log.warning("nothing to export")
68		return None
69	1
70
71		def export_lines(obj, ext='.yml', **kwargs):
72		"""Yield lines for an export in the specified format.
73
74		:param obj: Item, list of Items, or Document to export
75		:param ext: file extension to specify the output format
76
77		:raises: :class:`doorstop.common.DoorstopError` for unknown file formats
78
79		:return: lines generator
80	1
81	1	"""
82	1	gen = check(ext, get_lines_gen=True)
83		log.debug("yielding {} as lines of {}...".format(obj, ext))
84		yield from gen(obj, **kwargs)
85	1
86
87		def export_file(obj, path, ext=None, **kwargs):
88		"""Create a file object for an export in the specified format.
89
90		:param obj: Item, list of Items, or Document to export
91		:param path: output file location with desired extension
92		:param ext: file extension to override output path's extension
93
94		:raises: :class:`doorstop.common.DoorstopError` for unknown file formats
95
96		:return: path to created file
97	1
98	1	"""
99	1	ext = ext or os.path.splitext(path)[-1]
100	1	func = check(ext, get_file_func=True)
101	1	log.debug("converting %s to file format %s...", obj, ext)
102	1	try:
103	1	return func(obj, path, **kwargs)
104	1	except IOError:
105		msg = "unable to write to: {}".format(path)
106		raise common.DoorstopFileError(msg) from None
107	1
108
109		def _lines_yaml(obj, **_):
110		"""Yield lines for a YAML export.
111
112		:param obj: Item, list of Items, or Document to export
113
114		:return: iterator of lines of text
115	1
116		"""
117	1	for item in iter_items(obj):
118	1
119	1	data = {str(item.uid): item.data}
120		text = yaml.dump(data, default_flow_style=False, allow_unicode=True)
121		yield text
122	1
123
124		def _tabulate(obj, sep=LIST_SEP, auto=False):
125		"""Yield lines of header/data for tabular export.
126
127		:param obj: Item, list of Items, or Document to export
128		:param sep: string separating list values when joined in a string
129		:param auto: include placeholders for new items on import
130
131		:return: iterator of rows of data
132	1
133		"""
134	1	yield_header = True
135
136	1	for item in iter_items(obj):
137
138		data = item.data
139	1
140	1	# Yield header
141	1	if yield_header:
142	1	header = ['level', 'text', 'ref', 'links']
143	1	for value in sorted(data.keys()):
144	1	if value not in header:
145	1	header.append(value)
146		yield ['uid'] + header
147		yield_header = False
148	1
149	1	# Yield row
150	1	row = [item.uid]
151	1	for key in header:
		0 ignored issues – show introduced 2018-09-18 18:16 UTC by Report Bug Copy Issue Report The variable `header` does not seem to be defined for all execution paths. Loading history...
152		value = data.get(key)
153	1	if key == 'level':
154	1	# some levels are floats for YAML presentation
155		value = str(value)
156	1	elif key == 'links':
157	1	# separate identifiers with a delimiter
158		value = sep.join(uid.string for uid in item.links)
159	1	elif isinstance(value, str) and key not in ('reviewed',):
160	1	# remove sentence boundaries and line wrapping
161	1	value = item.get(key)
162	1	elif value is None:
163	1	value = ''
164		row.append(value)
165		yield row
166	1
167	1	# Yield placeholders for new items
168	1	if auto:
169		for _ in range(settings.PLACEHOLDER_COUNT):
170		yield [settings.PLACEHOLDER]
171	1
172
173		def _file_csv(obj, path, delimiter=',', auto=False):
174		"""Create a CSV file at the given path.
175
176		:param obj: Item, list of Items, or Document to export
177		:param path: location to export CSV file
178		:param delimiter: character to delimit fields
179		:param auto: include placeholders for new items on import
180
181		:return: path of created file
182	1
183	1	"""
184	1	with open(path, 'w', newline='', encoding='utf-8') as stream:
185	1	writer = csv.writer(stream, delimiter=delimiter)
186	1	for row in _tabulate(obj, auto=auto):
187		writer.writerow(row)
188		return path
189	1
190
191		def _file_tsv(obj, path, auto=False):
192		"""Create a TSV file at the given path.
193
194		:param obj: Item, list of Items, or Document to export
195		:param path: location to export TSV file
196		:param auto: include placeholders for new items on import
197
198		:return: path of created file
199	1
200		"""
201		return _file_csv(obj, path, delimiter='\t', auto=auto)
202	1
203
204		def _file_xlsx(obj, path, auto=False):
205		"""Create an XLSX file at the given path.
206
207		:param obj: Item, list of Items, or Document to export
208		:param path: location to export XLSX file
209		:param auto: include placeholders for new items on import
210
211		:return: path of created file
212	1
213	1	"""
214		workbook = _get_xlsx(obj, auto)
215	1	workbook.save(path)
216
217		return path
218	1
219
220		def _get_xlsx(obj, auto):
221		"""Create an XLSX workbook object.
222
223		:param obj: Item, list of Items, or Document to export
224		:param auto: include placeholders for new items on import
225
226		:return: new workbook
227	1
228	1	"""
229		col_widths: Dict[Any, float] = defaultdict(float)
230		col = 'A'
231	1
232	1	# Create a new workbook
233		workbook = openpyxl.Workbook()
234		worksheet = workbook.active
235	1
236	1	# Populate cells
237	1	for row, data in enumerate(_tabulate(obj, auto=auto), start=1):
238	1	for col_idx, value in enumerate(data, start=1):
239		cell = worksheet.cell(column=col_idx, row=row)
240
241	1	# wrap text in every cell
242		alignment = openpyxl.styles.Alignment(
243		vertical='top', horizontal='left', wrap_text=True
244	1	)
245		cell.alignment = alignment
246	1	# and bold header rows
247	1	if row == 1:
248	1	cell.font = openpyxl.styles.Font(bold=True)
249
250		# convert incompatible Excel types:
251		# http://pythonhosted.org/openpyxl/api.html#openpyxl.cell.Cell.value
252	1	if isinstance(value, (int, float, datetime.datetime)):
253	1	cell.value = value
254	1	else:
255		cell.value = str(value)
256
257	1	# track cell width
258		col_widths[col_idx] = max(col_widths[col_idx], _width(str(value)))
259
260	1	# Add filter up to the last column
261		col_letter = openpyxl.utils.get_column_letter(len(col_widths))
262		worksheet.auto_filter.ref = "A1:%s1" % col_letter
263	1
264	1	# Set column width based on column contents
265	1	for col in col_widths:
266		if col_widths[col] > XLSX_MAX_WIDTH:
267	1	width = XLSX_MAX_WIDTH
268	1	else:
269		width = col_widths[col] + XLSX_FILTER_PADDING
270		col_letter = openpyxl.utils.get_column_letter(col)
271	1	worksheet.column_dimensions[col_letter].width = width
272
273	1	# Freeze top row
274		worksheet.freeze_panes = worksheet.cell(row=2, column=1)
275
276	1	return workbook
277
278	1
279	1	def _width(text):
280		"""Get the maximum length in a multiline string."""
281	1	if text:
282		return max(len(line) for line in text.splitlines())
283		else:
284		return 0
285	1
286
287	1	# Mapping from file extension to lines generator
288		FORMAT_LINES = {'.yml': _lines_yaml}
289		# Mapping from file extension to file generator
290		FORMAT_FILE = {'.csv': _file_csv, '.tsv': _file_tsv, '.xlsx': _file_xlsx}
291	1	# Union of format dictionaries
292		FORMAT = dict(list(FORMAT_LINES.items()) + list(FORMAT_FILE.items())) # type: ignore
293
294	1
295		def check(ext, get_lines_gen=False, get_file_func=False):
296		"""Confirm an extension is supported for export.
297
298		:param get_lines_func: return a lines generator if available
299		:param get_file_func: return a file creator if available
300
301		:raises: :class:`doorstop.common.DoorstopError` for unknown formats
302
303		:return: function requested if available
304
305	1	"""
306	1	exts = ', '.join(ext for ext in FORMAT)
307	1	lines_exts = ', '.join(ext for ext in FORMAT_LINES)
308	1	file_exts = ', '.join(ext for ext in FORMAT_FILE)
309		fmt = "unknown {{}} format: {} (options: {{}})".format(ext or None)
310	1
311	1	if get_lines_gen:
312	1	try:
313	1	gen = FORMAT_LINES[ext]
314	1	except KeyError:
315	1	exc = DoorstopError(fmt.format("lines export", lines_exts))
316		raise exc from None
317	1	else:
318	1	log.debug("found lines generator for: {}".format(ext))
319		return gen
320	1
321	1	if get_file_func:
322	1	try:
323	1	func = FORMAT_FILE[ext]
324	1	except KeyError:
325	1	exc = DoorstopError(fmt.format("file export", file_exts))
326		raise exc from None
327	1	else:
328	1	log.debug("found file creator for: {}".format(ext))
329		return func
330	1
331	1	if ext not in FORMAT:
332	1	exc = DoorstopError(fmt.format("export", exts))
333		raise exc
334
335		return None
336

jacebrowning / doorstop

doorstop.core.exporter._tabulate() D last analyzed 2019-08-12 00:16 UTC

Complexity

Size

Duplication

Code Coverage

Importance

How to fix Complexity

Complexity

Duplication Side-by-Side

Filter issues like

doorstop.core.exporter._tabulate() D
last analyzed 2019-08-12 00:16 UTC