1
|
|
|
<?php |
2
|
|
|
/** |
3
|
|
|
* Created by PhpStorm. |
4
|
|
|
* User: nicolas |
5
|
|
|
* Date: 05/02/17 |
6
|
|
|
* Time: 14:32 |
7
|
|
|
*/ |
8
|
|
|
|
9
|
|
|
namespace Devgiants\Command; |
10
|
|
|
|
11
|
|
|
use Devgiants\Configuration\ConfigurationManager; |
12
|
|
|
use Devgiants\Configuration\ApplicationConfiguration as AppConf; |
13
|
|
|
use Devgiants\Exception\FailedStorageUploadException; |
14
|
|
|
use Devgiants\Model\ApplicationCommand; |
15
|
|
|
use Ifsnop\Mysqldump\Mysqldump; |
|
|
|
|
16
|
|
|
use Monolog\Handler\RotatingFileHandler; |
|
|
|
|
17
|
|
|
use Monolog\Logger; |
|
|
|
|
18
|
|
|
use Pimple\Container; |
|
|
|
|
19
|
|
|
use Symfony\Component\Console\Helper\ProgressBar; |
|
|
|
|
20
|
|
|
use Symfony\Component\Console\Input\InputInterface; |
|
|
|
|
21
|
|
|
use Symfony\Component\Console\Input\InputOption; |
|
|
|
|
22
|
|
|
use Symfony\Component\Console\Output\OutputInterface; |
|
|
|
|
23
|
|
|
use Symfony\Component\Filesystem\Filesystem; |
|
|
|
|
24
|
|
|
use Symfony\Component\Finder\Finder; |
|
|
|
|
25
|
|
|
use Symfony\Component\Yaml\Exception\ParseException; |
|
|
|
|
26
|
|
|
|
27
|
|
|
class BackupCommand extends ApplicationCommand { |
28
|
|
|
const FILE_OPTION = "file"; |
29
|
|
|
const ROOT_TEMP_PATH = "/tmp/websites-backups/"; |
30
|
|
|
const FILES = 'files'; |
31
|
|
|
const TEMP_PATHS = [ |
32
|
|
|
AppConf::DATABASE => self::ROOT_TEMP_PATH . "databases/", |
33
|
|
|
self::FILES => self::ROOT_TEMP_PATH . "files/" |
34
|
|
|
]; |
35
|
|
|
|
36
|
|
|
/** |
37
|
|
|
* RetrieveBackupCommand constructor. |
38
|
|
|
* |
39
|
|
|
* @param null|string $name |
40
|
|
|
* @param Container $container |
41
|
|
|
*/ |
42
|
|
|
public function __construct( $name, Container $container ) { |
43
|
|
|
parent::__construct( $name, $container ); |
44
|
|
|
} |
45
|
|
|
|
46
|
|
|
/** |
47
|
|
|
* @inheritdoc |
48
|
|
|
*/ |
49
|
|
|
protected function configure() { |
50
|
|
|
$this |
51
|
|
|
->setName( 'save' ) |
52
|
|
|
->setDescription( 'Backup sites accordingly to the YML configuration file provided' ) |
53
|
|
|
->setHelp( "This command allows you to save sites, maily by saving databases and files you chose" ) |
54
|
|
|
->addOption( self::FILE_OPTION, "f", InputOption::VALUE_REQUIRED, "The YML configuration file" ); |
55
|
|
|
} |
56
|
|
|
|
57
|
|
|
/** |
58
|
|
|
* @inheritdoc |
59
|
|
|
*/ |
60
|
|
|
protected function execute( InputInterface $input, OutputInterface $output ) { |
61
|
|
|
// Get conf file |
62
|
|
|
$ymlFile = $input->getOption( self::FILE_OPTION ); |
63
|
|
|
|
64
|
|
|
if ( $ymlFile !== null && is_file( $ymlFile ) ) { |
65
|
|
|
try { |
66
|
|
|
|
67
|
|
|
// Structures check and configuration loading |
68
|
|
|
$configurationManager = new ConfigurationManager( $ymlFile ); |
69
|
|
|
$configuration = $configurationManager->load(); |
70
|
|
|
|
71
|
|
|
$fs = new Filesystem(); |
72
|
|
|
|
73
|
|
|
// Defines 1 handler |
74
|
|
|
$this->log |
75
|
|
|
->pushHandler( new RotatingFileHandler( |
76
|
|
|
"{$configuration[AppConf::LOG_NODE[AppConf::NODE_NAME]]}/main.log", |
77
|
|
|
Logger::DEBUG ) |
78
|
|
|
); |
79
|
|
|
|
80
|
|
|
$this->log->addDebug( "" ); |
81
|
|
|
$this->log->addDebug( "----- START BACKUP SESSION -----" ); |
82
|
|
|
|
83
|
|
|
/********************************************* |
84
|
|
|
* Temp paths |
85
|
|
|
*/ |
86
|
|
|
foreach ( self::TEMP_PATHS as $tempPath ) { |
87
|
|
|
if ( ! file_exists( $tempPath ) ) { |
88
|
|
|
$this->log->addDebug( "Create temp folder : {$tempPath}" ); |
89
|
|
|
$output->write( "Create temp folders..." ); |
90
|
|
|
$fs->mkdir( $tempPath ); |
91
|
|
|
$output->write( "<info> DONE</info>" . PHP_EOL ); |
92
|
|
|
} |
93
|
|
|
} |
94
|
|
|
|
95
|
|
|
/********************************************* |
96
|
|
|
* Backup |
97
|
|
|
*/ |
98
|
|
|
$startTime = microtime(); |
|
|
|
|
99
|
|
|
|
100
|
|
|
$output->writeln( "Start sites backup" ); |
101
|
|
|
foreach ( $configuration['sites'] as $site => $siteConfiguration ) { |
102
|
|
|
|
103
|
|
|
// Create one log file per site |
104
|
|
|
$siteLog = new Logger( $site ); |
105
|
|
|
$siteLog |
106
|
|
|
->pushHandler( new RotatingFileHandler( |
107
|
|
|
"{$configuration[AppConf::LOG_NODE[AppConf::NODE_NAME]]}/{$site}.log", |
108
|
|
|
Logger::DEBUG ) |
109
|
|
|
); |
110
|
|
|
|
111
|
|
|
$siteLog->addDebug( "" ); |
112
|
|
|
$siteLog->addDebug( "----- START BACKUP SESSION -----" ); |
113
|
|
|
|
114
|
|
|
$output->writeln( "<fg=black;bg=yellow> - Site {$site}</>" ); |
115
|
|
|
$currentTimestamp = date( 'YmdHis' ); |
116
|
|
|
$siteLog->addDebug( "Start backup for site {$site}. Current timestamp: {$currentTimestamp}" ); |
117
|
|
|
|
118
|
|
|
/********************************************* |
119
|
|
|
* Pre-save commands |
120
|
|
|
*/ |
121
|
|
|
if ( count( $siteConfiguration[ AppConf::PRE_SAVE_COMMANDS ] ) > 0 ) { |
122
|
|
|
$output->writeln( " - Start pre-save commands" ); |
123
|
|
|
$siteLog->addDebug( "Pre save commands found." ); |
124
|
|
|
foreach ( $siteConfiguration['pre_save_commands'] as $command ) { |
125
|
|
|
$output->writeln( " - Run \"{$command}\"" ); |
126
|
|
|
$siteLog->addDebug( "Run \"{$command}\"" ); |
127
|
|
|
// TODO handle exec return |
128
|
|
|
exec( $command ); |
129
|
|
|
} |
130
|
|
|
$output->writeln( " - End pre-save commands" ); |
131
|
|
|
} |
132
|
|
|
|
133
|
|
|
/********************************************* |
134
|
|
|
* Databases |
135
|
|
|
*/ |
136
|
|
|
if ( isset( $siteConfiguration['database'] ) ) { |
137
|
|
|
$output->writeln( "<comment> - Databases</comment>" ); |
138
|
|
|
|
139
|
|
|
$dumpName = "{$site}_{$currentTimestamp}.sql.gz"; |
140
|
|
|
$dumpPath = self::TEMP_PATHS[ AppConf::DATABASE ] . $dumpName; |
141
|
|
|
$siteLog->addDebug( "Start database dump", [ 'dump_path' => $dumpPath ] ); |
142
|
|
|
|
143
|
|
|
try { |
144
|
|
|
$dump = new Mysqldump( |
145
|
|
|
"mysql:host={$siteConfiguration[AppConf::DATABASE][AppConf::SERVER]};dbname={$siteConfiguration[AppConf::DATABASE][AppConf::NAME]}", |
146
|
|
|
$siteConfiguration[ AppConf::DATABASE ][ AppConf::USER ], |
147
|
|
|
$siteConfiguration[ AppConf::DATABASE ][ AppConf::PASSWORD ], |
148
|
|
|
[ |
149
|
|
|
// TODO add compression as an option per site dump |
150
|
|
|
'compress' => Mysqldump::GZIP |
151
|
|
|
] |
152
|
|
|
); |
153
|
|
|
|
154
|
|
|
$output->write( " - Start database export and compression..." ); |
155
|
|
|
$siteLog->addDebug( "Start dump process" ); |
156
|
|
|
$dump->start( $dumpPath ); |
157
|
|
|
$output->write( "<info> DONE</info>" . PHP_EOL ); |
158
|
|
|
$siteLog->addDebug( "End dump process" ); |
159
|
|
|
} catch ( \Exception $e ) { |
160
|
|
|
echo 'mysqldump-php error: ' . $e->getMessage(); |
161
|
|
|
$output->writeln( "<error> - mysqldump-php error : {$e->getMessage()}</error>" ); |
162
|
|
|
$siteLog->addError( "MysqlDump PHP error :", [ |
163
|
|
|
'message' => $e->getMessage(), |
164
|
|
|
'code' => $e->getCode(), |
165
|
|
|
'file' => $e->getFile(), |
166
|
|
|
'line' => $e->getLine() |
167
|
|
|
] ); |
168
|
|
|
} |
169
|
|
|
} |
170
|
|
|
|
171
|
|
|
/********************************************* |
172
|
|
|
* Files |
173
|
|
|
*/ |
174
|
|
|
if ( isset( $siteConfiguration['files'] ) ) { |
175
|
|
|
$output->writeln( "<comment> - Files</comment>" ); |
176
|
|
|
$siteLog->addDebug( "Start files saving process" ); |
177
|
|
|
|
178
|
|
|
if ( is_dir( $siteConfiguration['files']['root_dir'] ) ) { |
179
|
|
|
try { |
180
|
|
|
// Build and create temp site path. Remove it before if needed to be sure it's empty |
181
|
|
|
$siteTempPath = self::TEMP_PATHS[ self::FILES ] . $site; |
182
|
|
|
if ( file_exists( $siteTempPath ) ) { |
183
|
|
|
$siteLog->addDebug( "Site temp path exists. Kills it", [ 'path' => $siteTempPath ] ); |
184
|
|
|
$fs->remove( $siteTempPath ); |
185
|
|
|
} |
186
|
|
|
$siteLog->addDebug( "Creates site temp path", [ 'path' => $siteTempPath ] ); |
187
|
|
|
$fs->mkdir( $siteTempPath ); |
188
|
|
|
|
189
|
|
|
$finder = new Finder(); |
190
|
|
|
|
191
|
|
|
$excludedFiles = []; |
192
|
|
|
foreach ( $siteConfiguration['files']['exclude'] as $relativeExcludedItem ) { |
193
|
|
|
$absoluteExcludedItem = "{$siteConfiguration['files']['root_dir']}{$relativeExcludedItem}"; |
194
|
|
|
|
195
|
|
|
// File case : directly add to exclude list |
196
|
|
|
if ( is_file( $absoluteExcludedItem ) ) { |
197
|
|
|
$siteLog->addDebug( "Add excluded file : {$absoluteExcludedItem}" ); |
198
|
|
|
$excludedFiles[] = $absoluteExcludedItem; |
199
|
|
|
} else { |
200
|
|
|
$siteLog->addDebug( "Handle excluded folder : {$absoluteExcludedItem}" ); |
201
|
|
|
foreach ( $finder->files()->followLinks()->in( $absoluteExcludedItem )->getIterator() as $excludedFile ) { |
202
|
|
|
$excludedFiles[] = $excludedFile->getRealPath(); |
203
|
|
|
} |
204
|
|
|
} |
205
|
|
|
} |
206
|
|
|
$siteLog->addDebug( "Final excluded files count : " . count( $excludedFiles ) ); |
207
|
|
|
|
208
|
|
|
// Copy all included folders recursively to temp path |
209
|
|
|
foreach ( $siteConfiguration['files']['include'] as $includedItem ) { |
210
|
|
|
$output->writeln( " - Start handling included {$siteConfiguration['files']['root_dir']}{$includedItem}" ); |
211
|
|
|
$siteLog->addDebug( "Start moving included files from {$siteConfiguration['files']['root_dir']}{$includedItem} to temp location" ); |
212
|
|
|
$includedFiles = $finder |
213
|
|
|
->files() |
214
|
|
|
->followLinks() |
215
|
|
|
->in( "{$siteConfiguration['files']['root_dir']}{$includedItem}" ); |
216
|
|
|
$output->writeln( " - " . count( $includedFiles ) . " files to copy" ); |
217
|
|
|
|
218
|
|
|
$filesProgressBar = new ProgressBar( $output, count( $includedFiles ) ); |
219
|
|
|
$filesProgressBar->setFormat( "very_verbose" ); |
220
|
|
|
$filesProgressBar->start(); |
221
|
|
|
|
222
|
|
|
foreach ( $includedFiles as $file ) { |
223
|
|
|
if ( ! in_array( $file->getRealPath(), $excludedFiles ) ) { |
224
|
|
|
$relativeFilePath = substr( $file->getRealPath(), strlen( $siteConfiguration['files']['root_dir'] ), strlen( $file->getRealPath() ) ); |
225
|
|
|
// Check file name length to avoid tar exception if name longer than 100 char |
226
|
|
|
if ( strlen( pathinfo( $relativeFilePath, PATHINFO_FILENAME ) ) < 100 ) { |
227
|
|
|
$fs->copy( $file->getRealPath(), "$siteTempPath/$relativeFilePath" ); |
228
|
|
|
} else { |
|
|
|
|
229
|
|
|
// TODO log it + make a list to let user know he have to change those name if he wants them to be saved |
230
|
|
|
} |
231
|
|
|
} |
232
|
|
|
$filesProgressBar->advance(); |
233
|
|
|
} |
234
|
|
|
|
235
|
|
|
$siteLog->addDebug( "Done" ); |
236
|
|
|
|
237
|
|
|
$filesProgressBar->finish(); |
238
|
|
|
$output->writeln( "" ); |
239
|
|
|
} |
240
|
|
|
|
241
|
|
|
|
242
|
|
|
// Compress files |
243
|
|
|
$archiveName = "{$site}_{$currentTimestamp}.tar.gz"; |
244
|
|
|
$archivePath = "{$siteTempPath}/{$archiveName}"; |
245
|
|
|
$archive = new \PharData( $archivePath ); |
246
|
|
|
$siteLog->addDebug( "Start compressing files to {$archivePath}" ); |
247
|
|
|
$archive->buildFromDirectory( $siteTempPath ); |
248
|
|
|
$siteLog->addDebug( "Done" ); |
249
|
|
|
} catch ( \Exception $e ) { |
250
|
|
|
$this->tools->maximumDetailsErrorHandling( $output, $siteLog, $e ); |
251
|
|
|
} |
252
|
|
|
} else { |
253
|
|
|
$output->writeln( "<error> - Error : \"{$siteConfiguration['files']['root_dir']}\" is not a valid directory path. Skip.</error>" ); |
254
|
|
|
$siteLog->addError( "\"{$siteConfiguration['files']['root_dir']}\" is not a valid directory path. Skip." ); |
255
|
|
|
} |
256
|
|
|
} |
257
|
|
|
|
258
|
|
|
/********************************************* |
259
|
|
|
* Store on external storages |
260
|
|
|
*/ |
261
|
|
|
try { |
262
|
|
|
$output->writeln( "<comment> - Storage</comment>" ); |
263
|
|
|
$siteLog->addDebug( "Start storage task" ); |
264
|
|
|
|
265
|
|
|
// Build storage array to loop on for saving |
266
|
|
|
$selectedStorages = []; |
267
|
|
|
if ( count( $siteConfiguration[ AppConf::BACKUP_STORAGES ] ) > 0 ) { |
268
|
|
|
foreach ( $siteConfiguration[ AppConf::BACKUP_STORAGES ] as $selectedStorageKey ) { |
269
|
|
|
$siteLog->addDebug( "Specific storage chosen : {$selectedStorageKey}" ); |
270
|
|
|
$selectedStorages[ $selectedStorageKey ] = $configuration[ AppConf::BACKUP_STORAGES ][ $selectedStorageKey ]; |
271
|
|
|
} |
272
|
|
|
} else { |
273
|
|
|
$siteLog->addDebug( 'All storage chosen' ); |
274
|
|
|
$selectedStorages = $configuration[ AppConf::BACKUP_STORAGES ]; |
275
|
|
|
} |
276
|
|
|
|
277
|
|
|
foreach ( $selectedStorages as $storageKey => $storage ) { |
278
|
|
|
|
279
|
|
|
$currentStorage = $this->tools->getStorageByType( $storage ); |
280
|
|
|
$siteLog->addDebug( "Start storage on {$storageKey} ({$storage[AppConf::STORAGE_TYPE]})" ); |
281
|
|
|
$output->write( " - Start storage on {$storageKey} ({$storage[AppConf::STORAGE_TYPE]})" . PHP_EOL ); |
282
|
|
|
// Connection |
283
|
|
|
$output->write( " - Trying connection..." ); |
284
|
|
|
$currentStorage->connect(); |
285
|
|
|
$siteLog->addDebug( "Connected to storage" ); |
286
|
|
|
$output->write( "<info> CONNECTED</info>" . PHP_EOL ); |
287
|
|
|
|
288
|
|
|
$remoteRootDir = "{$storage['root_dir']}/{$site}"; |
289
|
|
|
// TODO add trim start and trailing slashes (no more than once) |
290
|
|
|
$remoteDir = "{$remoteRootDir}/{$currentTimestamp}/"; |
291
|
|
|
$siteLog->addDebug( "Creates target remote directory : {$remoteDir}" ); |
292
|
|
|
$currentStorage->makeDir( $remoteDir ); |
293
|
|
|
// Dump |
294
|
|
|
if ( isset( $dumpName ) && isset( $dumpPath ) ) { |
295
|
|
|
$output->write( " - Start dump move..." ); |
296
|
|
|
if ( $currentStorage->put( $dumpPath, $remoteDir . $dumpName ) ) { |
|
|
|
|
297
|
|
|
$siteLog->addDebug( "Database dump moved to {$remoteDir}{$dumpName}" ); |
298
|
|
|
$output->write( "<info> DONE</info>" . PHP_EOL ); |
299
|
|
|
} else { |
300
|
|
|
throw new FailedStorageUploadException( "Dump {$dumpPath} failed to be uploaded to {$storageKey} ({$storage[AppConf::STORAGE_TYPE]}) to path {$remoteDir}{$dumpName}" ); |
301
|
|
|
} |
302
|
|
|
} |
303
|
|
|
|
304
|
|
|
// Archive |
305
|
|
|
if ( isset( $archiveName ) && isset( $archivePath ) ) { |
306
|
|
|
$output->write( " - Start archive move..." ); |
307
|
|
|
if ( $currentStorage->put( $archivePath, $remoteDir . $archiveName ) ) { |
|
|
|
|
308
|
|
|
$siteLog->addDebug( "Archive moved to {$remoteDir}{$archiveName}" ); |
309
|
|
|
$output->write( "<info> DONE</info>" . PHP_EOL ); |
310
|
|
|
} else { |
311
|
|
|
throw new FailedStorageUploadException( "Archive {$archivePath} failed to be uploaded to {$storageKey} ({$storage[AppConf::STORAGE_TYPE]}) to path {$remoteDir}{$dumpName}" ); |
312
|
|
|
} |
313
|
|
|
} |
314
|
|
|
|
315
|
|
|
/********************************************* |
316
|
|
|
* Retention |
317
|
|
|
*/ |
318
|
|
|
$siteLog->addDebug( "Handle retention" ); |
319
|
|
|
$currentStorage->handleRetention( $configuration[ AppConf::REMANENCE_NODE[ AppConf::NODE_NAME ] ], [ |
320
|
|
|
'root_dir' => $remoteRootDir |
321
|
|
|
] ); |
322
|
|
|
} |
323
|
|
|
|
324
|
|
|
$siteLog->addDebug( "End storage task" ); |
325
|
|
|
$output->writeln( " - End storage" ); |
326
|
|
|
|
327
|
|
|
} catch ( \Exception $e ) { |
328
|
|
|
$this->tools->maximumDetailsErrorHandling( $output, $siteLog, $e ); |
329
|
|
|
} |
330
|
|
|
|
331
|
|
|
/********************************************* |
332
|
|
|
* Post-save commands |
333
|
|
|
*/ |
334
|
|
|
if ( isset( $siteConfiguration['post_save_commands'] ) ) { |
335
|
|
|
$output->writeln( " - Start post-save commands" ); |
336
|
|
|
$siteLog->addDebug( "Post save commands found." ); |
337
|
|
|
foreach ( $siteConfiguration['post_save_commands'] as $command ) { |
338
|
|
|
$output->writeln( " - Run \"{$command}\"" ); |
339
|
|
|
$siteLog->addDebug( "Run \"{$command}\"" ); |
340
|
|
|
exec( $command ); |
341
|
|
|
} |
342
|
|
|
$output->writeln( " - End post-save commands" ); |
343
|
|
|
} |
344
|
|
|
} |
345
|
|
|
$output->writeln( "End sites backup" ); |
346
|
|
|
|
347
|
|
|
/********************************************* |
348
|
|
|
* Empty temp paths |
349
|
|
|
*/ |
350
|
|
|
$output->write( "Clear temp folders..." ); |
351
|
|
|
foreach ( self::TEMP_PATHS as $tempPath ) { |
352
|
|
|
$this->log->addDebug( "Clear temp path {$tempPath}" ); |
353
|
|
|
exec( "rm -rf $tempPath/*" ); |
354
|
|
|
} |
355
|
|
|
$output->write( "<info> DONE</info>" . PHP_EOL ); |
356
|
|
|
|
357
|
|
|
} catch ( ParseException $e ) { |
358
|
|
|
$output->writeln( "<error>Unable to parse the YAML string : {$e->getMessage()}</error>" ); |
359
|
|
|
$this->log->addError( "Unable to parse the YAML string : {$e->getMessage()}" ); |
360
|
|
|
} catch ( \Exception $e ) { |
361
|
|
|
$this->tools->maximumDetailsErrorHandling( $output, $this->log, $e ); |
362
|
|
|
} |
363
|
|
|
} else { |
364
|
|
|
$output->writeln( "<error>Filename is not correct : {$ymlFile}</error>" ); |
365
|
|
|
$this->log->addError( "Filename is not correct : {$ymlFile}" ); |
366
|
|
|
} |
367
|
|
|
$this->log->addDebug( "----- END BACKUP SESSION -----" ); |
368
|
|
|
} |
369
|
|
|
} |
The issue could also be caused by a filter entry in the build configuration. If the path has been excluded in your configuration, e.g.
excluded_paths: ["lib/*"]
, you can move it to the dependency path list as follows:For further information see https://scrutinizer-ci.com/docs/tools/php/php-scrutinizer/#list-dependency-paths