|
1
|
|
|
"""
|
|
2
|
|
|
Facilitates moving files from a specified directory and matching pattern to a destination directory
|
|
3
|
|
|
"""
|
|
4
|
|
|
# package to facilitate operating system operations
|
|
5
|
|
|
import os
|
|
6
|
|
|
|
|
7
|
|
|
# Custom classes specific to this package
|
|
8
|
|
|
from project_locale.localizations_common import LocalizationsCommon
|
|
9
|
|
|
from db_extractor.ExtractNeeds import ExtractNeeds
|
|
10
|
|
|
|
|
11
|
|
|
# get current script name
|
|
12
|
|
|
SCRIPT_NAME = os.path.basename(__file__).replace('.py', '')
|
|
13
|
|
|
|
|
14
|
|
|
# main execution logic
|
|
15
|
|
|
if __name__ == '__main__':
|
|
16
|
|
|
# instantiate Localizations Common class
|
|
17
|
|
|
class_lc = LocalizationsCommon()
|
|
18
|
|
|
# ensure all compiled localization files are in place (as needed for localized messages later)
|
|
19
|
|
|
class_lc.run_localization_compile()
|
|
20
|
|
|
# establish localization language to use
|
|
21
|
|
|
language_to_use = class_lc.get_region_language_to_use_from_operating_system()
|
|
22
|
|
|
# instantiate Logger class
|
|
23
|
|
|
c_en = ExtractNeeds(SCRIPT_NAME, language_to_use)
|
|
24
|
|
|
# load script configuration
|
|
25
|
|
|
c_en.load_configuration()
|
|
26
|
|
|
# initiate Logging sequence
|
|
27
|
|
|
c_en.initiate_logger_and_timer()
|
|
28
|
|
|
# reflect title and input parameters given values in the log
|
|
29
|
|
|
c_en.class_clam.listing_parameter_values(
|
|
30
|
|
|
c_en.class_ln.logger, c_en.timer, 'Database Extractor',
|
|
31
|
|
|
c_en.config['input_options'][SCRIPT_NAME], c_en.parameters)
|
|
32
|
|
|
# loading extracting sequence details
|
|
33
|
|
|
c_en.load_extraction_sequence_and_dependencies()
|
|
34
|
|
|
# validation of the extraction sequence file
|
|
35
|
|
|
if c_en.class_bnfe.validate_all_json_files(
|
|
36
|
|
|
c_en.class_ln.logger, c_en.timer, c_en.file_extract_sequence):
|
|
37
|
|
|
# cycling through the configurations
|
|
38
|
|
|
for seq_idx, crt_sequence in enumerate(c_en.file_extract_sequence):
|
|
39
|
|
|
can_proceed = c_en.class_bnfe.validate_all_json_files_current(
|
|
40
|
|
|
c_en.class_ln.logger, c_en.timer, crt_sequence, seq_idx,
|
|
41
|
|
|
c_en.source_systems, c_en.user_credentials)
|
|
42
|
|
|
if c_en.class_bn.fn_evaluate_dict_values(can_proceed):
|
|
43
|
|
|
c_en.class_dbt.connect_to_database(
|
|
44
|
|
|
c_en.class_ln.logger, c_en.timer, c_en.class_bnfe.connection_details)
|
|
45
|
|
|
if c_en.class_dbt.connection is not None:
|
|
46
|
|
|
# instantiate DB connection handler
|
|
47
|
|
|
cursor = c_en.class_dbt.connection.cursor()
|
|
48
|
|
|
for crt_query in crt_sequence['queries']:
|
|
49
|
|
|
can_proceed_q = c_en.class_bnfe.validate_extraction_query(
|
|
50
|
|
|
c_en.class_ln.logger, c_en.timer, crt_query)
|
|
51
|
|
|
if can_proceed_q:
|
|
52
|
|
|
the_query = c_en.load_query(crt_query)
|
|
53
|
|
|
for crt_session in crt_query['sessions']:
|
|
54
|
|
|
dict__child__parent__grand_parent = c_en.pack_three_levels(
|
|
55
|
|
|
crt_session, crt_query, crt_sequence)
|
|
56
|
|
|
crt_session['start-iso-weekday'] = \
|
|
57
|
|
|
c_en.set_default_starting_weekday(
|
|
58
|
|
|
dict__child__parent__grand_parent)
|
|
59
|
|
|
dict__child__parent__grand_parent = c_en.pack_three_levels(
|
|
60
|
|
|
crt_session, crt_query, crt_sequence)
|
|
61
|
|
|
if 'parameters' in crt_session:
|
|
62
|
|
|
crt_session['parameters-handling-rules'] = \
|
|
63
|
|
|
c_en.set_default_parameter_rules(
|
|
64
|
|
|
dict__child__parent__grand_parent)
|
|
65
|
|
|
can_proceed_ses = \
|
|
66
|
|
|
c_en.class_bnfe.validate_query_session(
|
|
67
|
|
|
c_en.class_ln.logger, crt_session)
|
|
68
|
|
|
crt_session['extract-behaviour'] = \
|
|
69
|
|
|
c_en.class_bnfe.fn_set_extract_behaviour(crt_session)
|
|
70
|
|
|
dict__child__parent__grand_parent = c_en.pack_three_levels(
|
|
71
|
|
|
crt_session, crt_query, crt_sequence)
|
|
72
|
|
|
extraction_required = c_en.evaluate_if_extraction_is_required(
|
|
73
|
|
|
dict__child__parent__grand_parent)
|
|
74
|
|
|
if can_proceed_ses and extraction_required:
|
|
75
|
|
|
dict_prepared = {
|
|
76
|
|
|
'query': the_query,
|
|
77
|
|
|
'session': crt_session,
|
|
78
|
|
|
}
|
|
79
|
|
|
stats = c_en.extract_query_to_result_set(
|
|
80
|
|
|
c_en.class_ln.logger, cursor, dict_prepared)
|
|
81
|
|
|
if stats['rows_counted'] > 0:
|
|
82
|
|
|
dict__child__parent__grand_parent = c_en.pack_three_levels(
|
|
83
|
|
|
crt_session, crt_query, crt_sequence)
|
|
84
|
|
|
c_en.result_set_to_disk_file(
|
|
85
|
|
|
c_en.class_ln.logger, stats,
|
|
86
|
|
|
dict__child__parent__grand_parent)
|
|
87
|
|
|
c_en.close_cursor(c_en.class_ln.logger, cursor)
|
|
88
|
|
|
c_en.close_connection(c_en.class_ln.logger)
|
|
89
|
|
|
# just final message
|
|
90
|
|
|
c_en.class_bn.fn_final_message(c_en.class_ln.logger, c_en.parameters.output_log_file,
|
|
91
|
|
|
c_en.timer.timers.total(SCRIPT_NAME))
|
|
92
|
|
|
|
danielgp /
db-extractor
