| 1 |  |  | #!/usr/bin/env python3 | 
            
                                                                                                            
                            
            
                                    
            
            
                | 2 |  |  | # -*- coding: future_fstrings -*- | 
            
                                                                                                            
                            
            
                                    
            
            
                | 3 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 4 |  |  | """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 5 |  |  | Transfer script | 
            
                                                                                                            
                            
            
                                    
            
            
                | 6 |  |  | """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 7 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 8 |  |  | import sys | 
            
                                                                                                            
                            
            
                                    
            
            
                | 9 |  |  | from db_sync_tool.utility import mode, system, helper, output | 
            
                                                                                                            
                            
            
                                    
            
            
                | 10 |  |  | from db_sync_tool.database import utility as database_utility | 
            
                                                                                                            
                            
            
                                    
            
            
                | 11 |  |  | from db_sync_tool.remote import utility, client, rsync | 
            
                                                                                                            
                            
            
                                    
            
            
                | 12 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 13 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 14 |  |  | def transfer_origin_database_dump(): | 
            
                                                                                                            
                            
            
                                    
            
            
                | 15 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 16 |  |  |     Transfer the origin database dump files | 
            
                                                                                                            
                            
            
                                    
            
            
                | 17 |  |  |     :return: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 18 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 19 |  |  |     if not mode.is_import(): | 
            
                                                                                                            
                            
            
                                    
            
            
                | 20 |  |  |         if mode.get_sync_mode() == mode.SyncMode.RECEIVER: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 21 |  |  |             get_origin_database_dump(helper.get_dump_dir(mode.Client.TARGET)) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 22 |  |  |             system.check_target_configuration() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 23 |  |  |         elif mode.get_sync_mode() == mode.SyncMode.SENDER: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 24 |  |  |             system.check_target_configuration() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 25 |  |  |             put_origin_database_dump(helper.get_dump_dir(mode.Client.ORIGIN)) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 26 |  |  |             utility.remove_origin_database_dump() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 27 |  |  |         elif mode.get_sync_mode() == mode.SyncMode.PROXY: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 28 |  |  |             helper.create_local_temporary_data_dir() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 29 |  |  |             get_origin_database_dump(system.default_local_sync_path) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 30 |  |  |             system.check_target_configuration() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 31 |  |  |             put_origin_database_dump(system.default_local_sync_path) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 32 |  |  |         elif mode.get_sync_mode() == mode.SyncMode.SYNC_REMOTE or mode.get_sync_mode() == mode.SyncMode.SYNC_LOCAL: | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 33 |  |  |             system.check_target_configuration() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 34 |  |  |         elif system.config['is_same_client']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 35 |  |  |             utility.remove_origin_database_dump(True) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 36 |  |  |     else: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 37 |  |  |         system.check_target_configuration() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 38 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 39 |  |  |  | 
            
                                                                                                            
                            
            
                                                                    
                                                                                                        
            
            
                | 40 |  | View Code Duplication | def get_origin_database_dump(target_path): | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 41 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 42 |  |  |     Downloading the origin database dump files | 
            
                                                                                                            
                            
            
                                    
            
            
                | 43 |  |  |     :param target_path: String | 
            
                                                                                                            
                            
            
                                    
            
            
                | 44 |  |  |     :return: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 45 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 46 |  |  |     output.message( | 
            
                                                                                                            
                            
            
                                    
            
            
                | 47 |  |  |         output.Subject.ORIGIN, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 48 |  |  |         'Downloading database dump', | 
            
                                                                                                            
                            
            
                                    
            
            
                | 49 |  |  |         True | 
            
                                                                                                            
                            
            
                                    
            
            
                | 50 |  |  |     ) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 51 |  |  |     if mode.get_sync_mode() != mode.SyncMode.PROXY: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 52 |  |  |         helper.check_and_create_dump_dir(mode.Client.TARGET, target_path) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 53 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 54 |  |  |     if not system.config['dry_run']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 55 |  |  |         _remotepath = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz' | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 56 |  |  |         _localpath = target_path | 
            
                                                                                                            
                            
            
                                    
            
            
                | 57 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 58 |  |  |         if system.config['use_rsync']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 59 |  |  |             rsync.run_rsync_command( | 
            
                                                                                                            
                            
            
                                    
            
            
                | 60 |  |  |                 remote_client=mode.Client.ORIGIN, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 61 |  |  |                 origin_path=_remotepath, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 62 |  |  |                 target_path=_localpath, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 63 |  |  |                 origin_ssh=system.config[mode.Client.ORIGIN]['user'] + '@' + system.config[mode.Client.ORIGIN]['host'] | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 64 |  |  |             ) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 65 |  |  |         else: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 66 |  |  |             # | 
            
                                                                                                            
                            
            
                                    
            
            
                | 67 |  |  |             # Download speed problems | 
            
                                                                                                            
                            
            
                                    
            
            
                | 68 |  |  |             # https://github.com/paramiko/paramiko/issues/60 | 
            
                                                                                                            
                            
            
                                    
            
            
                | 69 |  |  |             # | 
            
                                                                                                            
                            
            
                                    
            
            
                | 70 |  |  |             sftp = get_sftp_client(client.ssh_client_origin) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 71 |  |  |             sftp.get(helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz', | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 72 |  |  |                      target_path + database_utility.database_dump_file_name + '.tar.gz', download_status) | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 73 |  |  |             sftp.close() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 74 |  |  |             if not system.config['mute']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 75 |  |  |                 print('') | 
            
                                                                                                            
                            
            
                                    
            
            
                | 76 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 77 |  |  |     utility.remove_origin_database_dump() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 78 |  |  |  | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 79 |  |  |  | 
            
                                                                        
                            
            
                                    
            
            
                | 80 |  |  | def download_status(sent, size): | 
            
                                                                        
                            
            
                                    
            
            
                | 81 |  |  |     """ | 
            
                                                                        
                            
            
                                    
            
            
                | 82 |  |  |     Printing the download status information | 
            
                                                                        
                            
            
                                    
            
            
                | 83 |  |  |     :param sent: Float | 
            
                                                                        
                            
            
                                    
            
            
                | 84 |  |  |     :param size: Float | 
            
                                                                        
                            
            
                                    
            
            
                | 85 |  |  |     :return: | 
            
                                                                        
                            
            
                                    
            
            
                | 86 |  |  |     """ | 
            
                                                                        
                            
            
                                    
            
            
                | 87 |  |  |     if not system.config['mute']: | 
            
                                                                        
                            
            
                                    
            
            
                | 88 |  |  |         sent_mb = round(float(sent) / 1024 / 1024, 1) | 
            
                                                                        
                            
            
                                    
            
            
                | 89 |  |  |         size = round(float(size) / 1024 / 1024, 1) | 
            
                                                                        
                            
            
                                    
            
            
                | 90 |  |  |         sys.stdout.write( | 
            
                                                                        
                            
            
                                    
            
            
                | 91 |  |  |             output.Subject.ORIGIN + output.CliFormat.BLACK + '[REMOTE]' + output.CliFormat.ENDC + " Status: {0} MB of {1} MB downloaded". | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                        
                            
            
                                    
            
            
                | 92 |  |  |             format(sent_mb, size, )) | 
            
                                                                        
                            
            
                                    
            
            
                | 93 |  |  |         sys.stdout.write('\r') | 
            
                                                                                                            
                            
            
                                    
            
            
                | 94 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 95 |  |  |  | 
            
                                                                                                            
                            
            
                                                                    
                                                                                                        
            
            
                | 96 |  | View Code Duplication | def put_origin_database_dump(origin_path): | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 97 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 98 |  |  |     Uploading the origin database dump file | 
            
                                                                                                            
                            
            
                                    
            
            
                | 99 |  |  |     :param origin_path: String | 
            
                                                                                                            
                            
            
                                    
            
            
                | 100 |  |  |     :return: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 101 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 102 |  |  |     if mode.get_sync_mode() == mode.SyncMode.PROXY: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 103 |  |  |         _subject = output.Subject.LOCAL | 
            
                                                                                                            
                            
            
                                    
            
            
                | 104 |  |  |     else: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 105 |  |  |         _subject = output.Subject.ORIGIN | 
            
                                                                                                            
                            
            
                                    
            
            
                | 106 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 107 |  |  |     output.message( | 
            
                                                                                                            
                            
            
                                    
            
            
                | 108 |  |  |         _subject, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 109 |  |  |         'Uploading database dump', | 
            
                                                                                                            
                            
            
                                    
            
            
                | 110 |  |  |         True | 
            
                                                                                                            
                            
            
                                    
            
            
                | 111 |  |  |     ) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 112 |  |  |     helper.check_and_create_dump_dir(mode.Client.TARGET, helper.get_dump_dir(mode.Client.TARGET)) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 113 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 114 |  |  |     if not system.config['dry_run']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 115 |  |  |         _localpath = origin_path + database_utility.database_dump_file_name + '.tar.gz' | 
            
                                                                                                            
                            
            
                                    
            
            
                | 116 |  |  |         _remotepath = helper.get_dump_dir(mode.Client.TARGET) + '/' | 
            
                                                                                                            
                            
            
                                    
            
            
                | 117 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 118 |  |  |         if system.config['use_rsync']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 119 |  |  |             rsync.run_rsync_command( | 
            
                                                                                                            
                            
            
                                    
            
            
                | 120 |  |  |                 remote_client=mode.Client.TARGET, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 121 |  |  |                 origin_path=_localpath, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 122 |  |  |                 target_path=_remotepath, | 
            
                                                                                                            
                            
            
                                    
            
            
                | 123 |  |  |                 target_ssh=system.config[mode.Client.TARGET]['user'] + '@' + system.config[mode.Client.TARGET]['host'] | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 124 |  |  |             ) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 125 |  |  |         else: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 126 |  |  |             # | 
            
                                                                                                            
                            
            
                                    
            
            
                | 127 |  |  |             # Download speed problems | 
            
                                                                                                            
                            
            
                                    
            
            
                | 128 |  |  |             # https://github.com/paramiko/paramiko/issues/60 | 
            
                                                                                                            
                            
            
                                    
            
            
                | 129 |  |  |             # | 
            
                                                                                                            
                            
            
                                    
            
            
                | 130 |  |  |             sftp = get_sftp_client(client.ssh_client_target) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 131 |  |  |             sftp.put(origin_path + database_utility.database_dump_file_name + '.tar.gz', | 
            
                                                                                                            
                            
            
                                    
            
            
                | 132 |  |  |                      helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name + '.tar.gz', | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 133 |  |  |                      upload_status) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 134 |  |  |             sftp.close() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 135 |  |  |             if not system.config['mute']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 136 |  |  |                 print('') | 
            
                                                                                                            
                            
            
                                    
            
            
                | 137 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 138 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 139 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 140 |  |  | def upload_status(sent, size): | 
            
                                                                                                            
                            
            
                                    
            
            
                | 141 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 142 |  |  |     Printing the upload status information | 
            
                                                                                                            
                            
            
                                    
            
            
                | 143 |  |  |     :param sent: Float | 
            
                                                                                                            
                            
            
                                    
            
            
                | 144 |  |  |     :param size: Float | 
            
                                                                                                            
                            
            
                                    
            
            
                | 145 |  |  |     :return: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 146 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 147 |  |  |     if not system.config['mute']: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 148 |  |  |         sent_mb = round(float(sent) / 1024 / 1024, 1) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 149 |  |  |         size = round(float(size) / 1024 / 1024, 1) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 150 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 151 |  |  |         if (mode.get_sync_mode() == mode.SyncMode.PROXY): | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 152 |  |  |             _subject = output.Subject.LOCAL | 
            
                                                                                                            
                            
            
                                    
            
            
                | 153 |  |  |         else: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 154 |  |  |             _subject = output.Subject.ORIGIN + output.CliFormat.BLACK + '[LOCAL]' + output.CliFormat.ENDC | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                                                                            
                            
            
                                    
            
            
                | 155 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 156 |  |  |         sys.stdout.write( | 
            
                                                                                                            
                            
            
                                    
            
            
                | 157 |  |  |             _subject + " Status: {0} MB of {1} MB uploaded". | 
            
                                                                                                            
                            
            
                                    
            
            
                | 158 |  |  |             format(sent_mb, size, )) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 159 |  |  |         sys.stdout.write('\r') | 
            
                                                                                                            
                            
            
                                    
            
            
                | 160 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 161 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 162 |  |  | def get_sftp_client(ssh_client): | 
            
                                                                                                            
                            
            
                                    
            
            
                | 163 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 164 |  |  |  | 
            
                                                                                                            
                            
            
                                    
            
            
                | 165 |  |  |     :param ssh_client: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 166 |  |  |     :return: | 
            
                                                                                                            
                            
            
                                    
            
            
                | 167 |  |  |     """ | 
            
                                                                                                            
                            
            
                                    
            
            
                | 168 |  |  |     sftp = ssh_client.open_sftp() | 
            
                                                                                                            
                            
            
                                    
            
            
                | 169 |  |  |     sftp.get_channel().settimeout(client.default_timeout) | 
            
                                                                                                            
                            
            
                                    
            
            
                | 170 |  |  |     return sftp | 
            
                                                                                                            
                                                                
            
                                    
            
            
                | 171 |  |  |  | 
                            
                    |  |  |  | 
                                                                                        
                                                                                     | 
            
                                                        
            
                                    
            
            
                | 172 |  |  |  | 
            
                        
This check looks for lines that are too long. You can specify the maximum line length.