TableauHyperApiExtraLogic.TableauHyperApiExtraLogic.fn_run_create_hyper_file_from_csv() - Code Metrics - Inspection of "introduced a new function to focus on convertion a..." - danielgp/tableau-hyper-management - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Push — master ( 3e515d...19e016 )

by Daniel

created 2019-11-21 00:26 UTC

TableauHyperApiExtraLogic.fn_run_create_hyper_file_from_csv() A

↳ Parent: TableauHyperApiExtraLogic

Complexity

Conditions

Size

Total Lines	56
Code Lines	34

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	4
eloc	34
nop	4
dl	0
loc	56
rs	9.064
c	0
b	0
f	0

How to fix Long Method

import csv

from BasicNeeds import BasicNeeds as cls_bn
from TypeDetermination import TypeDetermination
from datetime import datetime,time
from tableauhyperapi import HyperProcess, Telemetry, \
    Connection, CreateMode, \
    NOT_NULLABLE, NULLABLE, SqlType, TableDefinition, \
    Inserter, \
    escape_name, escape_string_literal, \
    TableName, \
    HyperException, \
    Timestamp


class TableauHyperApiExtraLogic:

    def fn_build_hyper_columns_for_csv(given_file_name, csv_field_separator, detected_csv_structure, verbose):
        list_hyper_table_columns_to_return = []
        for current_field_structure in detected_csv_structure:
            list_hyper_table_columns_to_return.append(current_field_structure['order'])
            current_column_type = TableauHyperApiExtraLogic.fn_convert_to_hyper_types(current_field_structure['type'])
            cls_bn.fn_optional_print(cls_bn, verbose, 'Column '
                                     + str(current_field_structure['order']) + ' having name "'
                                     + current_field_structure['name'] + '" and type "'
                                     + current_field_structure['type'] + '" will become "'
                                     + str(current_column_type) + '"')
            list_hyper_table_columns_to_return[current_field_structure['order']] = TableDefinition.Column(
                name=current_field_structure['name'],
                type=current_column_type,
                nullability=NULLABLE
            )
        return list_hyper_table_columns_to_return

    def fn_convert_and_validate_content(self, crt_value, crt_type):
        if crt_value == '':
            return None
        else:
            if crt_type == 'int':
                return int(crt_value)
            elif crt_type == 'float-USA':
                return float(crt_value)
            elif crt_type == 'date-iso8601':
                tm = datetime.strptime(crt_value, '%Y-%m-%d')
                return datetime(tm.year, tm.month, tm.day)
            elif crt_type == 'date-USA':
                tm = datetime.strptime(crt_value, '%m/%d/%Y')
                return datetime(tm.year, tm.month, tm.day)
            elif crt_type == 'time-24':
                tm = datetime.strptime(crt_value, '%H:%M:%S')
                return time(tm.hour, tm.minute, tm.second)
            elif crt_type == 'time-24-us':
                tm = datetime.strptime(crt_value, '%H:%M:%S.%f')
                return time(tm.hour, tm.minute, tm.second, tm.microsecond)
            elif crt_type == 'time-USA':
                tm = datetime.strptime(crt_value, '%I:%M:%S')
                return time(tm.hour, tm.minute, tm.second)
            elif crt_type == 'datetime-iso8601':
                tm = datetime.fromisoformat(crt_value)
                return Timestamp(tm.year, tm.month, tm.day, tm.hour, tm.minute, tm.second)
            elif crt_type == 'datetime-iso8601-us':
                tm = datetime.fromisoformat(crt_value)
                return Timestamp(tm.year, tm.month, tm.day, tm.hour, tm.minute, tm.second, tm.microsecond)
            else:
                return crt_value.replace('"', '\\"')

    def fn_convert_to_hyper_types(given_type):
        switcher = {
            'empty': SqlType.text(),
            'int': SqlType.big_int(),
            'float-USA': SqlType.double(),
            'date-iso8601': SqlType.date(),
            'date-USA': SqlType.date(),
            'time-24': SqlType.time(),
            'time-24-us': SqlType.time(),
            'time-USA': SqlType.time(),
            'datetime-iso8601': SqlType.timestamp(),
            'str': SqlType.text()
        }
        identified_type = switcher.get(given_type)
        if identified_type is None:
            identified_type = SqlType.text()
        return identified_type

    def fn_rebuild_csv_content_for_hyper(given_file_name, csv_field_separator, detected_fields_type, verbose):
        csv_content_for_hyper = []
        with open(given_file_name, newline='') as csv_file:
            csv_object = csv.DictReader(csv_file, delimiter=csv_field_separator)
            # parse rows with index
            for row_idx, row_content in enumerate(csv_object):
                csv_content_for_hyper.append(row_idx)
                csv_content_for_hyper[row_idx] = []
                print_prefix = 'On the row ' + str((row_idx + 1))
                # parse all columns with index
                for col_idx, column_name in enumerate(csv_object.fieldnames):
                    csv_content_for_hyper[row_idx].append(col_idx)
                    csv_content_for_hyper[row_idx][col_idx] = \
                        self.fn_convert_and_validate_content(self,

                                                             row_content[csv_object.fieldnames[col_idx]],
                                                             detected_fields_type[col_idx]['type'])
                    cls_bn.fn_optional_print(cls_bn, verbose, print_prefix + ' column ' + str(col_idx)
                                             + ' having the name [' + csv_object.fieldnames[col_idx] + '] '
                                             + ' has the value <' + row_content[csv_object.fieldnames[col_idx]]
                                             + '> which was interpreted as <<'
                                             + str(csv_content_for_hyper[row_idx][col_idx])
                                             + '>>')
        return csv_content_for_hyper

    def fn_run_create_hyper_file_from_csv(input_csv_file,
                                          csv_field_separator,
                                          output_hyper_file,
                                          verbose):
        detected_csv_structure = TypeDetermination.fn_detect_csv_structure(TypeDetermination,
                                                                           input_csv_file,
                                                                           csv_field_separator,
                                                                           verbose)
        hyper_table_columns = TableauHyperApiExtraLogic.fn_build_hyper_columns_for_csv(input_csv_file,
                                                                                       csv_field_separator,
                                                                                       detected_csv_structure,
                                                                                       verbose)
        # Starts the Hyper Process with telemetry enabled/disabled to send data to Tableau or not
        # To opt in, simply set telemetry=Telemetry.SEND_USAGE_DATA_TO_TABLEAU.
        # To opt out, simply set telemetry=Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU.
        with HyperProcess(telemetry=Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU) as hyper:
            # Creates new Hyper file <output_hyper_file>
            # Replaces file with CreateMode.CREATE_AND_REPLACE if it already exists.
            with Connection(endpoint=hyper.endpoint,
                            database=output_hyper_file,
                            create_mode=CreateMode.CREATE_AND_REPLACE) as hyper_connection:
                hyper_connection.catalog.create_schema("Extract")
                hyper_table = TableDefinition(
                    name=TableName("Extract", "Extract"),
                    columns=hyper_table_columns
                )
                hyper_connection.catalog.create_table(table_definition=hyper_table)
                print("The connection to the Hyper engine file has been created.")
                '''
                VERDICT: does not work as DOUBLE or INT are not accepting empty values... :-(
                print("I am about to execute command: " 
                    + f"COPY {hyper_table.table_name} from {escape_string_literal(input_csv_file)} with "
                    f"(format csv, NULL 'NULL', delimiter '{csv_field_separator}', header)")
                # Load all rows into "Customers" table from the CSV file.
                # `execute_command` executes a SQL statement and returns the impacted row count.
                count_in_target_table = hyper_connection.execute_command(
                    command=f"COPY {hyper_table.table_name} from {escape_string_literal(input_csv_file)} with "
                    f"(format csv, NULL 'NULL', delimiter '{csv_field_separator}', header)")
                print(f"The number of rows in table {hyper_table.table_name} is {count_in_target_table}.")
                '''
                # The rows to insert into the <hyper_table> table.
                data_to_insert = TableauHyperApiExtraLogic.fn_rebuild_csv_content_for_hyper(input_csv_file,
                                                                                            csv_field_separator,
                                                                                            detected_csv_structure,
                                                                                            verbose)
                # Execute the actual insert
                with Inserter(hyper_connection, hyper_table) as hyper_inserter:
                    hyper_inserter.add_rows(rows=data_to_insert)
                    hyper_inserter.execute()
                # Number of rows in the <hyper_table> table.
                # `execute_scalar_query` is for executing a query that returns exactly one row with one column.
                row_count = hyper_connection.\
                    execute_scalar_query(query=f"SELECT COUNT(*) FROM {hyper_table.table_name}")
                print(f"The number of rows in table {hyper_table.table_name} is {row_count}.")
            print("The connection to the Hyper file has been closed.")
        print("The Hyper process has been shut down.")

1			import csv
2
3			from BasicNeeds import BasicNeeds as cls_bn
4			from TypeDetermination import TypeDetermination
5			from datetime import datetime,time
6			from tableauhyperapi import HyperProcess, Telemetry, \
7			Connection, CreateMode, \
8			NOT_NULLABLE, NULLABLE, SqlType, TableDefinition, \
9			Inserter, \
10			escape_name, escape_string_literal, \
11			TableName, \
12			HyperException, \
13			Timestamp
14
15
16			class TableauHyperApiExtraLogic:
17
18			def fn_build_hyper_columns_for_csv(given_file_name, csv_field_separator, detected_csv_structure, verbose):
19			list_hyper_table_columns_to_return = []
20			for current_field_structure in detected_csv_structure:
21			list_hyper_table_columns_to_return.append(current_field_structure['order'])
22			current_column_type = TableauHyperApiExtraLogic.fn_convert_to_hyper_types(current_field_structure['type'])
23			cls_bn.fn_optional_print(cls_bn, verbose, 'Column '
24			+ str(current_field_structure['order']) + ' having name "'
25			+ current_field_structure['name'] + '" and type "'
26			+ current_field_structure['type'] + '" will become "'
27			+ str(current_column_type) + '"')
28			list_hyper_table_columns_to_return[current_field_structure['order']] = TableDefinition.Column(
29			name=current_field_structure['name'],
30			type=current_column_type,
31			nullability=NULLABLE
32			)
33			return list_hyper_table_columns_to_return
34
35			def fn_convert_and_validate_content(self, crt_value, crt_type):
36			if crt_value == '':
37			return None
38			else:
39			if crt_type == 'int':
40			return int(crt_value)
41			elif crt_type == 'float-USA':
42			return float(crt_value)
43			elif crt_type == 'date-iso8601':
44			tm = datetime.strptime(crt_value, '%Y-%m-%d')
45			return datetime(tm.year, tm.month, tm.day)
46			elif crt_type == 'date-USA':
47			tm = datetime.strptime(crt_value, '%m/%d/%Y')
48			return datetime(tm.year, tm.month, tm.day)
49			elif crt_type == 'time-24':
50			tm = datetime.strptime(crt_value, '%H:%M:%S')
51			return time(tm.hour, tm.minute, tm.second)
52			elif crt_type == 'time-24-us':
53			tm = datetime.strptime(crt_value, '%H:%M:%S.%f')
54			return time(tm.hour, tm.minute, tm.second, tm.microsecond)
55			elif crt_type == 'time-USA':
56			tm = datetime.strptime(crt_value, '%I:%M:%S')
57			return time(tm.hour, tm.minute, tm.second)
58			elif crt_type == 'datetime-iso8601':
59			tm = datetime.fromisoformat(crt_value)
60			return Timestamp(tm.year, tm.month, tm.day, tm.hour, tm.minute, tm.second)
61			elif crt_type == 'datetime-iso8601-us':
62			tm = datetime.fromisoformat(crt_value)
63			return Timestamp(tm.year, tm.month, tm.day, tm.hour, tm.minute, tm.second, tm.microsecond)
64			else:
65			return crt_value.replace('"', '\\"')
66
67			def fn_convert_to_hyper_types(given_type):
68			switcher = {
69			'empty': SqlType.text(),
70			'int': SqlType.big_int(),
71			'float-USA': SqlType.double(),
72			'date-iso8601': SqlType.date(),
73			'date-USA': SqlType.date(),
74			'time-24': SqlType.time(),
75			'time-24-us': SqlType.time(),
76			'time-USA': SqlType.time(),
77			'datetime-iso8601': SqlType.timestamp(),
78			'str': SqlType.text()
79			}
80			identified_type = switcher.get(given_type)
81			if identified_type is None:
82			identified_type = SqlType.text()
83			return identified_type
84
85			def fn_rebuild_csv_content_for_hyper(given_file_name, csv_field_separator, detected_fields_type, verbose):
86			csv_content_for_hyper = []
87			with open(given_file_name, newline='') as csv_file:
88			csv_object = csv.DictReader(csv_file, delimiter=csv_field_separator)
89			# parse rows with index
90			for row_idx, row_content in enumerate(csv_object):
91			csv_content_for_hyper.append(row_idx)
92			csv_content_for_hyper[row_idx] = []
93			print_prefix = 'On the row ' + str((row_idx + 1))
94			# parse all columns with index
95			for col_idx, column_name in enumerate(csv_object.fieldnames):
96			csv_content_for_hyper[row_idx].append(col_idx)
97			csv_content_for_hyper[row_idx][col_idx] = \
98			self.fn_convert_and_validate_content(self,
			0 ignored issues – show Comprehensibility Best Practice introduced 2019-11-21 00:27 UTC by Report Bug Copy Issue Report The variable `self` does not seem to be defined. Loading history...
99			row_content[csv_object.fieldnames[col_idx]],
100			detected_fields_type[col_idx]['type'])
101			cls_bn.fn_optional_print(cls_bn, verbose, print_prefix + ' column ' + str(col_idx)
102			+ ' having the name [' + csv_object.fieldnames[col_idx] + '] '
103			+ ' has the value <' + row_content[csv_object.fieldnames[col_idx]]
104			+ '> which was interpreted as <<'
105			+ str(csv_content_for_hyper[row_idx][col_idx])
106			+ '>>')
107			return csv_content_for_hyper
108
109			def fn_run_create_hyper_file_from_csv(input_csv_file,
110			csv_field_separator,
111			output_hyper_file,
112			verbose):
113			detected_csv_structure = TypeDetermination.fn_detect_csv_structure(TypeDetermination,
114			input_csv_file,
115			csv_field_separator,
116			verbose)
117			hyper_table_columns = TableauHyperApiExtraLogic.fn_build_hyper_columns_for_csv(input_csv_file,
118			csv_field_separator,
119			detected_csv_structure,
120			verbose)
121			# Starts the Hyper Process with telemetry enabled/disabled to send data to Tableau or not
122			# To opt in, simply set telemetry=Telemetry.SEND_USAGE_DATA_TO_TABLEAU.
123			# To opt out, simply set telemetry=Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU.
124			with HyperProcess(telemetry=Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU) as hyper:
125			# Creates new Hyper file <output_hyper_file>
126			# Replaces file with CreateMode.CREATE_AND_REPLACE if it already exists.
127			with Connection(endpoint=hyper.endpoint,
128			database=output_hyper_file,
129			create_mode=CreateMode.CREATE_AND_REPLACE) as hyper_connection:
130			hyper_connection.catalog.create_schema("Extract")
131			hyper_table = TableDefinition(
132			name=TableName("Extract", "Extract"),
133			columns=hyper_table_columns
134			)
135			hyper_connection.catalog.create_table(table_definition=hyper_table)
136			print("The connection to the Hyper engine file has been created.")
137			'''
138			VERDICT: does not work as DOUBLE or INT are not accepting empty values... :-(
139			print("I am about to execute command: "
140			+ f"COPY {hyper_table.table_name} from {escape_string_literal(input_csv_file)} with "
141			f"(format csv, NULL 'NULL', delimiter '{csv_field_separator}', header)")
142			# Load all rows into "Customers" table from the CSV file.
143			# `execute_command` executes a SQL statement and returns the impacted row count.
144			count_in_target_table = hyper_connection.execute_command(
145			command=f"COPY {hyper_table.table_name} from {escape_string_literal(input_csv_file)} with "
146			f"(format csv, NULL 'NULL', delimiter '{csv_field_separator}', header)")
147			print(f"The number of rows in table {hyper_table.table_name} is {count_in_target_table}.")
148			'''
149			# The rows to insert into the <hyper_table> table.
150			data_to_insert = TableauHyperApiExtraLogic.fn_rebuild_csv_content_for_hyper(input_csv_file,
151			csv_field_separator,
152			detected_csv_structure,
153			verbose)
154			# Execute the actual insert
155			with Inserter(hyper_connection, hyper_table) as hyper_inserter:
156			hyper_inserter.add_rows(rows=data_to_insert)
157			hyper_inserter.execute()
158			# Number of rows in the <hyper_table> table.
159			# `execute_scalar_query` is for executing a query that returns exactly one row with one column.
160			row_count = hyper_connection.\
161			execute_scalar_query(query=f"SELECT COUNT(*) FROM {hyper_table.table_name}")
162			print(f"The number of rows in table {hyper_table.table_name} is {row_count}.")
163			print("The connection to the Hyper file has been closed.")
164			print("The Hyper process has been shut down.")

danielgp / tableau-hyper-management

Push — master ( 3e515d...19e016 )

TableauHyperApiExtraLogic.fn_run_create_hyper_file_from_csv() A

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like